You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:24:25 UTC
[buildstream] branch richardmaw/shell-multi-stage created (now
f7954ef)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at f7954ef Add to NEWS
This branch includes the following new commits:
new 8b89d61 dev-requirements: Bump pytest requirement for tmp_path
new 5525920 manual.py: Add BST_VIRTUAL_DIRECTORY flag
new bc111ff element: Add BST_STAGE_INTEGRATES Element flag
new 6b9a93f element: use BST_STAGE_INTEGRATES in prepare_sandbox and assemble
new 173adb7 scriptelement: Skip integration in stage if BST_STAGE_INTEGRATES not set
new 5e57926 elements: Port build and script kinds to BST_STAGE_INTEGRATES
new 21b6ba3 _stream: Move shell logic from Element and support multiple elements
new 10a5243 element: Remove dead _shell method
new f87b7fd cli: Support multiple elements in `bst shell`
new 2441ab3 tests: Move integration.bst to subdirectory
new 87b660f tests: Test multiple-element runtime and workspace shells
new f7954ef Add to NEWS
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 03/12: element: Add BST_STAGE_INTEGRATES Element flag
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bc111ff9c1061c2e9bd53b0176aef92e89b5ab0d
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 13:07:44 2018 +0000
element: Add BST_STAGE_INTEGRATES Element flag
This commit introduces BST_STAGE_INTEGRATES as a compatibility flag,
so Elements can set it to false
to signal that integrate_dependency_artifacts should be called.
This will be needed to implement staging multiple elements,
since the integration commands for all elements need to be called
after all dependencies are staged.
---
buildstream/element.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 46 insertions(+), 3 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index 2f724f4..13140a1 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -176,6 +176,15 @@ class Element(Plugin):
*Since: 1.4*
"""
+ BST_STAGE_INTEGRATES = True
+ """If True, Element.stage() is expected to run integration commands for its dependencies.
+ If False, Element.integrate_dependency_artifacts() is called after Element.stage().
+
+ If True then multi-element sandboxes can't include this element.
+
+ *Since: 1.4*
+ """
+
def __init__(self, context, project, meta, plugin_conf):
self.__cache_key_dict = None # Dict for cache key calculation
@@ -291,11 +300,12 @@ class Element(Plugin):
raise ImplError("element plugin '{kind}' does not implement configure_sandbox()".format(
kind=self.get_kind()))
- def stage(self, sandbox):
+ def stage(self, sandbox, *, visited=None):
"""Stage inputs into the sandbox directories
Args:
sandbox (:class:`.Sandbox`): The build sandbox
+ visited (dict or None): Skip Elements included here and add any traversed
Raises:
(:class:`.ElementError`): When the element raises an error
@@ -304,10 +314,42 @@ class Element(Plugin):
directory with data. This is done either by staging :class:`.Source`
objects, by staging the artifacts of the elements this element depends
on, or both.
+
+ The visited parameter does not need to be declared unless BST_STAGE_INTEGRATES is set to False.
+ If BST_STAGE_INTEGRATES is False then visited should be passed to calls to dependencies().
"""
raise ImplError("element plugin '{kind}' does not implement stage()".format(
kind=self.get_kind()))
+ def integrate_dependency_artifacts(self, sandbox, scope, *, visited=None):
+ """Integrate element dependencies in scope
+
+ Args:
+ sandbox (:class:`.Sandbox`): The build sandbox
+ scope (:class:`.Scope`): The scope to stage dependencies in
+ visited (dict or None): Skip Elements included here and add any traversed
+
+ This is primarily a convenience wrapper around
+ :func:`Element.integrate() <buildstream.element.Element.stage_artifact>`
+ which takes care of integrating all the dependencies in `scope`.
+
+ This defaults to just calling integrate() on every dependency.
+ If an Element needs to defer, reorder or change the selection of which Elements
+ should have their integration commands run, then this should be overridden.
+
+ This method is not guaranteed to be called when BST_STAGE_INTEGRATES is True
+ when staging this Element in a build scope,
+ but implementations of `stage()` may opt to call it in these circumstances.
+
+ *Since: 1.4*
+ """
+ if visited is None:
+ visited = {}
+
+ with sandbox.batch(SandboxFlags.NONE, label="Integrating sandbox"):
+ for dep in self.dependencies(scope, visited=visited):
+ dep.integrate(sandbox)
+
def prepare(self, sandbox):
"""Run one-off preparation commands.
@@ -656,7 +698,7 @@ class Element(Plugin):
return link_result.combine(copy_result)
def stage_dependency_artifacts(self, sandbox, scope, *, path=None,
- include=None, exclude=None, orphans=True):
+ include=None, exclude=None, orphans=True, visited=None):
"""Stage element dependencies in scope
This is primarily a convenience wrapper around
@@ -671,6 +713,7 @@ class Element(Plugin):
include (list): An optional list of domains to include files from
exclude (list): An optional list of domains to exclude files from
orphans (bool): Whether to include files not spoken for by split domains
+ visited (dict or None): Skip Elements included here and add any traversed
Raises:
(:class:`.ElementError`): If any of the dependencies in `scope` have not
@@ -686,7 +729,7 @@ class Element(Plugin):
if self.__can_build_incrementally() and workspace.last_successful:
old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful)
- for dep in self.dependencies(scope):
+ for dep in self.dependencies(scope, visited=visited):
# If we are workspaced, and we therefore perform an
# incremental build, we must ensure that we update the mtimes
# of any files created by our dependencies since the last
[buildstream] 08/12: element: Remove dead _shell method
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 10a5243b35d6ae1d4a8fc72c203f543cfcdd0c10
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Thu Oct 25 11:27:31 2018 +0100
element: Remove dead _shell method
---
buildstream/element.py | 89 +++++---------------------------------------------
1 file changed, 9 insertions(+), 80 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index 35c305f..cec1e9f 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -75,7 +75,6 @@ Class Reference
import os
import re
import stat
-import copy
from collections import OrderedDict
from collections.abc import Mapping
import contextlib
@@ -1382,7 +1381,7 @@ class Element(Plugin):
# is used to stage things by the `bst checkout` codepath
#
@contextmanager
- def _prepare_sandbox(self, scope, directory, shell=False, integrate=True):
+ def _prepare_sandbox(self, scope, directory, integrate=True):
# bst shell and bst checkout require a local sandbox.
bare_directory = True if directory else False
with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
@@ -1393,20 +1392,15 @@ class Element(Plugin):
# Stage something if we need it
if not directory:
- if shell and scope == Scope.BUILD:
- self.stage(sandbox)
- if not self.BST_STAGE_INTEGRATES:
+ # Stage deps in the sandbox root
+ with self.timed_activity("Staging dependencies", silent_nested=True):
+ self.stage_dependency_artifacts(sandbox, scope)
+
+ # Run any integration commands provided by the dependencies
+ # once they are all staged and ready
+ if integrate:
+ with self.timed_activity("Integrating sandbox"):
self.integrate_dependency_artifacts(sandbox, scope)
- else:
- # Stage deps in the sandbox root
- with self.timed_activity("Staging dependencies", silent_nested=True):
- self.stage_dependency_artifacts(sandbox, scope)
-
- # Run any integration commands provided by the dependencies
- # once they are all staged and ready
- if integrate:
- with self.timed_activity("Integrating sandbox"):
- self.integrate_dependency_artifacts(sandbox, scope)
yield sandbox
@@ -1885,71 +1879,6 @@ class Element(Plugin):
# Notify successful upload
return True
- # _shell():
- #
- # Connects the terminal with a shell running in a staged
- # environment
- #
- # Args:
- # scope (Scope): Either BUILD or RUN scopes are valid, or None
- # directory (str): A directory to an existing sandbox, or None
- # mounts (list): A list of (str, str) tuples, representing host/target paths to mount
- # isolate (bool): Whether to isolate the environment like we do in builds
- # prompt (str): A suitable prompt string for PS1
- # command (list): An argv to launch in the sandbox
- #
- # Returns: Exit code
- #
- # If directory is not specified, one will be staged using scope
- def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
-
- with self._prepare_sandbox(scope, directory, shell=True) as sandbox:
- environment = self.get_environment()
- environment = copy.copy(environment)
- flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
-
- # Fetch the main toplevel project, in case this is a junctioned
- # subproject, we want to use the rules defined by the main one.
- context = self._get_context()
- project = context.get_toplevel_project()
- shell_command, shell_environment, shell_host_files = project.get_shell_config()
-
- if prompt is not None:
- environment['PS1'] = prompt
-
- # Special configurations for non-isolated sandboxes
- if not isolate:
-
- # Open the network, and reuse calling uid/gid
- #
- flags |= SandboxFlags.NETWORK_ENABLED | SandboxFlags.INHERIT_UID
-
- # Apply project defined environment vars to set for a shell
- for key, value in _yaml.node_items(shell_environment):
- environment[key] = value
-
- # Setup any requested bind mounts
- if mounts is None:
- mounts = []
-
- for mount in shell_host_files + mounts:
- if not os.path.exists(mount.host_path):
- if not mount.optional:
- self.warn("Not mounting non-existing host file: {}".format(mount.host_path))
- else:
- sandbox.mark_directory(mount.path)
- sandbox._set_mount_source(mount.path, mount.host_path)
-
- if command:
- argv = [arg for arg in command]
- else:
- argv = shell_command
-
- self.status("Running command", detail=" ".join(argv))
-
- # Run shells with network enabled and readonly root.
- return sandbox.run(argv, flags, env=environment)
-
# _open_workspace():
#
# "Open" a workspace for this element
[buildstream] 10/12: tests: Move integration.bst to subdirectory
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2441ab395d973ef760dbf9e7c68f646778cea0a4
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 16:44:28 2018 +0000
tests: Move integration.bst to subdirectory
---
tests/integration/project/elements/{ => shell}/integration.bst | 0
tests/integration/shell.py | 3 ++-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/tests/integration/project/elements/integration.bst b/tests/integration/project/elements/shell/integration.bst
similarity index 100%
rename from tests/integration/project/elements/integration.bst
rename to tests/integration/project/elements/shell/integration.bst
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index 68535bf..d4e672a 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -345,11 +345,12 @@ def test_sysroot(cli, tmpdir, datafiles):
# Test system integration commands can access devices in /dev
+@pytest.mark.integration
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
def test_integration_devices(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
- element_name = 'integration.bst'
+ element_name = 'shell/integration.bst'
result = execute_shell(cli, project, ["true"], element=element_name)
assert result.exit_code == 0
[buildstream] 01/12: dev-requirements: Bump pytest requirement for
tmp_path
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8b89d618a45c386a90782ae965a35abf670228f9
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Wed Nov 21 16:21:06 2018 +0000
dev-requirements: Bump pytest requirement for tmp_path
tmp_path was actually introduced in the 3.9.x series,
but it slipped by when the pytest version was bumped before
by pip opting to fetch a version which was newer than necessary.
---
dev-requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 380e734..20e8834 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,7 +1,7 @@
coverage == 4.4.0
pep8
pylint == 2.1.1
-pytest >= 3.8
+pytest >= 3.9.1
pytest-cov >= 2.5.0
pytest-datafiles
pytest-env
[buildstream] 09/12: cli: Support multiple elements in `bst shell`
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f87b7fd386e2ec7ba85104c887d9dd8b490b0a4f
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 13:46:16 2018 +0000
cli: Support multiple elements in `bst shell`
---
buildstream/_frontend/cli.py | 42 +++++++++++++++++++++++++++++++-----------
1 file changed, 31 insertions(+), 11 deletions(-)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 0b2ebc7..ebbd25b 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -573,7 +573,7 @@ def show(app, elements, deps, except_, order, format_):
##################################################################
@cli.command(short_help="Shell into an element's sandbox environment")
@click.option('--build', '-b', 'build_', is_flag=True, default=False,
- help='Stage dependencies and sources to build')
+ help='Stage dependencies and sources to build the first element')
@click.option('--sysroot', '-s', default=None,
type=click.Path(exists=True, file_okay=False, readable=True),
help="An existing sysroot")
@@ -582,11 +582,11 @@ def show(app, elements, deps, except_, order, format_):
help="Mount a file or directory into the sandbox")
@click.option('--isolate', is_flag=True, default=False,
help='Create an isolated build sandbox')
-@click.argument('element',
- type=click.Path(readable=False))
-@click.argument('command', type=click.STRING, nargs=-1)
+@click.argument('elements',
+ type=click.Path(readable=False), nargs=-1,
+ metavar="[ELEMENT]... [--] [COMMAND]...")
@click.pass_obj
-def shell(app, element, sysroot, mount, isolate, build_, command):
+def shell(app, elements, sysroot, mount, isolate, build_):
"""Run a command in the target element's sandbox environment
This will stage a temporary sysroot for running the target
@@ -600,6 +600,15 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
directory or with a checkout of the given target, in order
to use a specific sysroot.
+ Multiple element target names ending in .bst may be provided,
+ optionally followed by the command to run in the shell.
+
+ The first argument that doesn't end in .bst is assumed to be the beginning of COMMAND.
+
+ A -- argument may be used to disambiguate between element target names and command arguments
+ and should be used when being driven as part of a script, to prevent mis-parsing,
+ in addition to a -- before the element list.
+
If no COMMAND is specified, the default is to attempt
to run an interactive shell.
"""
@@ -607,13 +616,22 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
from .._project import HostMount
from .._pipeline import PipelineSelection
- if build_:
- scope = Scope.BUILD
- else:
- scope = Scope.RUN
+ targets = []
+ command = []
+ for i, arg in enumerate(elements):
+ if arg == '--':
+ command = elements[i + 1:]
+ break
+ if not arg.endswith('.bst'):
+ command = elements[i:]
+ break
+ targets.append(arg)
+
+ if not targets:
+ raise AppError('No elements specified to open a shell in')
with app.initialized():
- dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
+ dependencies = app.stream.load_selection(targets, selection=PipelineSelection.NONE)
element = dependencies[0]
prompt = app.shell_prompt(element)
mounts = [
@@ -621,7 +639,9 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
for host_path, path in mount
]
try:
- exitcode = app.stream.shell([(element, scope)], prompt,
+ elements = tuple((e, Scope.BUILD if i == 0 and build_ else Scope.RUN)
+ for (i, e) in enumerate(dependencies))
+ exitcode = app.stream.shell(elements, prompt,
directory=sysroot,
mounts=mounts,
isolate=isolate,
[buildstream] 06/12: elements: Port build and script kinds to
BST_STAGE_INTEGRATES
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5e5792642ec043aacea3169402b7e10881a35c74
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 13:39:55 2018 +0000
elements: Port build and script kinds to BST_STAGE_INTEGRATES
This is mostly just marking which elements work as expected
from changes to their base class.
Junction and Filter elements expect overriding stage to be sufficient
to prevent it doing anything in those contexts,
and since we're not intending to deprecate BST_STAGE_INTEGRATES elements
there's no harm in them continuing to do so
when they aren't expected to work in contexts we require
BST_STAGE_INTEGRATES to be False.
---
buildstream/buildelement.py | 15 ++++++++-------
buildstream/plugins/elements/autotools.py | 2 ++
buildstream/plugins/elements/cmake.py | 2 ++
buildstream/plugins/elements/compose.py | 12 ++++++++----
buildstream/plugins/elements/distutils.py | 3 ++-
buildstream/plugins/elements/filter.py | 2 +-
buildstream/plugins/elements/import.py | 7 ++++++-
buildstream/plugins/elements/junction.py | 2 +-
buildstream/plugins/elements/make.py | 2 ++
buildstream/plugins/elements/makemaker.py | 3 ++-
buildstream/plugins/elements/manual.py | 2 ++
buildstream/plugins/elements/meson.py | 2 ++
buildstream/plugins/elements/modulebuild.py | 3 ++-
buildstream/plugins/elements/pip.py | 3 ++-
buildstream/plugins/elements/qmake.py | 2 ++
buildstream/plugins/elements/script.py | 3 +++
buildstream/plugins/elements/stack.py | 8 +++++++-
17 files changed, 54 insertions(+), 19 deletions(-)
diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py
index f1b13e7..67d9594 100644
--- a/buildstream/buildelement.py
+++ b/buildstream/buildelement.py
@@ -211,17 +211,18 @@ class BuildElement(Element):
self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
collect=self.get_variable('install-root'))
- def stage(self, sandbox):
+ def stage(self, sandbox, *, visited=None):
+ assert not self.BST_STAGE_INTEGRATES or visited is None
# Stage deps in the sandbox root
with self.timed_activity("Staging dependencies", silent_nested=True):
- self.stage_dependency_artifacts(sandbox, Scope.BUILD)
+ self.stage_dependency_artifacts(sandbox, Scope.BUILD, visited=visited)
- # Run any integration commands provided by the dependencies
- # once they are all staged and ready
- with sandbox.batch(SandboxFlags.NONE, label="Integrating sandbox"):
- for dep in self.dependencies(Scope.BUILD):
- dep.integrate(sandbox)
+ if self.BST_STAGE_INTEGRATES:
+ # Run any integration commands provided by the dependencies
+ # once they are all staged and ready
+ with self.timed_activity("Integrating sandbox"):
+ self.integrate_dependency_artifacts(sandbox, Scope.BUILD)
# Stage sources in the build root
self.stage_sources(sandbox, self.get_variable('build-root'))
diff --git a/buildstream/plugins/elements/autotools.py b/buildstream/plugins/elements/autotools.py
index 43fe7ad..8ea24c1 100644
--- a/buildstream/plugins/elements/autotools.py
+++ b/buildstream/plugins/elements/autotools.py
@@ -62,6 +62,8 @@ from buildstream import BuildElement
class AutotoolsElement(BuildElement):
# Supports virtual directories (required for remote execution)
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/cmake.py b/buildstream/plugins/elements/cmake.py
index de9aa96..3ce2d42 100644
--- a/buildstream/plugins/elements/cmake.py
+++ b/buildstream/plugins/elements/cmake.py
@@ -61,6 +61,8 @@ from buildstream import BuildElement
class CMakeElement(BuildElement):
# Supports virtual directories (required for remote execution)
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index d61a324..1c34480 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -58,6 +58,9 @@ class ComposeElement(Element):
# This plugin has been modified to avoid the use of Sandbox.get_directory
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
+
def configure(self, node):
self.node_validate(node, [
'integrate', 'include', 'exclude', 'include-orphans'
@@ -86,7 +89,10 @@ class ComposeElement(Element):
def configure_sandbox(self, sandbox):
pass
- def stage(self, sandbox):
+ def stage(self, sandbox, *, visited=None):
+ pass
+
+ def integrate_dependency_artifacts(self, sandbox, scope, *, visited=None):
pass
def assemble(self, sandbox):
@@ -122,9 +128,7 @@ class ComposeElement(Element):
snapshot = set(vbasedir.list_relative_paths())
vbasedir.mark_unmodified()
- with sandbox.batch(0):
- for dep in self.dependencies(Scope.BUILD):
- dep.integrate(sandbox)
+ super().integrate_dependency_artifacts(sandbox, Scope.BUILD)
if require_split:
# Calculate added, modified and removed files
diff --git a/buildstream/plugins/elements/distutils.py b/buildstream/plugins/elements/distutils.py
index 20a2d67..efef493 100644
--- a/buildstream/plugins/elements/distutils.py
+++ b/buildstream/plugins/elements/distutils.py
@@ -36,7 +36,8 @@ from buildstream import BuildElement
# Element implementation for the python 'distutils' kind.
class DistutilsElement(BuildElement):
- pass
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/filter.py b/buildstream/plugins/elements/filter.py
index 6723253..3d882c7 100644
--- a/buildstream/plugins/elements/filter.py
+++ b/buildstream/plugins/elements/filter.py
@@ -97,7 +97,7 @@ class FilterElement(Element):
def configure_sandbox(self, sandbox):
pass
- def stage(self, sandbox):
+ def stage(self, sandbox, *, visited=None):
pass
def assemble(self, sandbox):
diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py
index ec491a0..a65994a 100644
--- a/buildstream/plugins/elements/import.py
+++ b/buildstream/plugins/elements/import.py
@@ -43,6 +43,8 @@ class ImportElement(BuildElement):
# This plugin has been modified to avoid the use of Sandbox.get_directory
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
def configure(self, node):
self.source = self.node_subst_member(node, 'source')
@@ -64,7 +66,10 @@ class ImportElement(BuildElement):
def configure_sandbox(self, sandbox):
pass
- def stage(self, sandbox):
+ def stage(self, sandbox, *, visited=None):
+ pass
+
+ def integrate_dependency_artifacts(self, sandbox, scope, *, visited=None):
pass
def assemble(self, sandbox):
diff --git a/buildstream/plugins/elements/junction.py b/buildstream/plugins/elements/junction.py
index 7f98173..05d5bff 100644
--- a/buildstream/plugins/elements/junction.py
+++ b/buildstream/plugins/elements/junction.py
@@ -152,7 +152,7 @@ class JunctionElement(Element):
def configure_sandbox(self, sandbox):
raise PipelineError("Cannot build junction elements")
- def stage(self, sandbox):
+ def stage(self, sandbox, *, visited=None):
raise PipelineError("Cannot stage junction elements")
def generate_script(self):
diff --git a/buildstream/plugins/elements/make.py b/buildstream/plugins/elements/make.py
index a22de16..5a3387e 100644
--- a/buildstream/plugins/elements/make.py
+++ b/buildstream/plugins/elements/make.py
@@ -43,6 +43,8 @@ from buildstream import BuildElement
class MakeElement(BuildElement):
# Supports virtual directories (required for remote execution)
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/makemaker.py b/buildstream/plugins/elements/makemaker.py
index 8ad9799..f2cd7a7 100644
--- a/buildstream/plugins/elements/makemaker.py
+++ b/buildstream/plugins/elements/makemaker.py
@@ -36,7 +36,8 @@ from buildstream import BuildElement
# Element implementation for the 'makemaker' kind.
class MakeMakerElement(BuildElement):
- pass
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/manual.py b/buildstream/plugins/elements/manual.py
index 8f951ab..393a009 100644
--- a/buildstream/plugins/elements/manual.py
+++ b/buildstream/plugins/elements/manual.py
@@ -38,6 +38,8 @@ from buildstream import BuildElement
class ManualElement(BuildElement):
# Supports virtual directories (required for remote execution)
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/meson.py b/buildstream/plugins/elements/meson.py
index 0d5ca00..312ae40 100644
--- a/buildstream/plugins/elements/meson.py
+++ b/buildstream/plugins/elements/meson.py
@@ -58,6 +58,8 @@ from buildstream import BuildElement
class MesonElement(BuildElement):
# Supports virtual directories (required for remote execution)
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/modulebuild.py b/buildstream/plugins/elements/modulebuild.py
index d619a1d..89ec908 100644
--- a/buildstream/plugins/elements/modulebuild.py
+++ b/buildstream/plugins/elements/modulebuild.py
@@ -36,7 +36,8 @@ from buildstream import BuildElement
# Element implementation for the 'modulebuild' kind.
class ModuleBuildElement(BuildElement):
- pass
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/pip.py b/buildstream/plugins/elements/pip.py
index 02b593a..f2cf377 100644
--- a/buildstream/plugins/elements/pip.py
+++ b/buildstream/plugins/elements/pip.py
@@ -36,7 +36,8 @@ from buildstream import BuildElement
# Element implementation for the 'pip' kind.
class PipElement(BuildElement):
- pass
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/qmake.py b/buildstream/plugins/elements/qmake.py
index d6776b0..314219d 100644
--- a/buildstream/plugins/elements/qmake.py
+++ b/buildstream/plugins/elements/qmake.py
@@ -38,6 +38,8 @@ from buildstream import BuildElement
class QMakeElement(BuildElement):
# Supports virtual directories (required for remote execution)
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
# Plugin entry point
diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py
index 4e422c5..5ad93dd 100644
--- a/buildstream/plugins/elements/script.py
+++ b/buildstream/plugins/elements/script.py
@@ -42,6 +42,9 @@ import buildstream
class ScriptElement(buildstream.ScriptElement):
# pylint: disable=attribute-defined-outside-init
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
+
def configure(self, node):
for n in self.node_get_member(node, list, 'layout', []):
dst = self.node_subst_member(n, 'destination')
diff --git a/buildstream/plugins/elements/stack.py b/buildstream/plugins/elements/stack.py
index 138afed..54e540c 100644
--- a/buildstream/plugins/elements/stack.py
+++ b/buildstream/plugins/elements/stack.py
@@ -33,6 +33,9 @@ class StackElement(Element):
# This plugin has been modified to avoid the use of Sandbox.get_directory
BST_VIRTUAL_DIRECTORY = True
+ # This plugin has been modified to permit calling integration after staging
+ BST_STAGE_INTEGRATES = False
+
def configure(self, node):
pass
@@ -47,7 +50,10 @@ class StackElement(Element):
def configure_sandbox(self, sandbox):
pass
- def stage(self, sandbox):
+ def stage(self, sandbox, *, visited=None):
+ pass
+
+ def integrate_dependency_artifacts(self, sandbox, scope, *, visited=None):
pass
def assemble(self, sandbox):
[buildstream] 07/12: _stream: Move shell logic from Element and
support multiple elements
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 21b6ba36a7fdc7eb68dc0fede901f18b4b7fcff2
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 13:42:52 2018 +0000
_stream: Move shell logic from Element and support multiple elements
There's redundancy in pre-staging sandbox creation logic
with Element._sandbox doing something similar,
but hopefully not too much.
---
buildstream/_frontend/app.py | 2 +-
buildstream/_frontend/cli.py | 2 +-
buildstream/_stream.py | 127 +++++++++++++++++++++++++++++++++++++++----
3 files changed, 117 insertions(+), 14 deletions(-)
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index 4094eec..671eefa 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -599,7 +599,7 @@ class App():
click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True)
try:
prompt = self.shell_prompt(element)
- self.stream.shell(element, Scope.BUILD, prompt, isolate=True)
+ self.stream.shell([(element, Scope.BUILD)], prompt, isolate=True)
except BstError as e:
click.echo("Error while attempting to create interactive shell: {}".format(e), err=True)
elif choice == 'log':
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index ef0dadb..0b2ebc7 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -621,7 +621,7 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
for host_path, path in mount
]
try:
- exitcode = app.stream.shell(element, scope, prompt,
+ exitcode = app.stream.shell([(element, scope)], prompt,
directory=sysroot,
mounts=mounts,
isolate=isolate,
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index 6a3d2c7..aeda136 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -25,15 +25,17 @@ import stat
import shlex
import shutil
import tarfile
-from contextlib import contextmanager
+from contextlib import contextmanager, ExitStack
from tempfile import TemporaryDirectory
from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
from ._message import Message, MessageType
-from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
from ._pipeline import Pipeline, PipelineSelection
+from ._platform import Platform
+from .sandbox._config import SandboxConfig
+from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
from . import utils, _yaml, _site
-from . import Scope, Consistency
+from . import SandboxFlags, Scope, Consistency
# Stream()
@@ -117,8 +119,7 @@ class Stream():
# Run a shell
#
# Args:
- # element (Element): An Element object to run the shell for
- # scope (Scope): The scope for the shell (Scope.BUILD or Scope.RUN)
+ # elements (List of (Element, Scope) tuples): Elements to run the shell for and their dependency scopes.
# prompt (str): The prompt to display in the shell
# directory (str): A directory where an existing prestaged sysroot is expected, or None
# mounts (list of HostMount): Additional directories to mount into the sandbox
@@ -128,7 +129,7 @@ class Stream():
# Returns:
# (int): The exit code of the launched shell
#
- def shell(self, element, scope, prompt, *,
+ def shell(self, elements, prompt, *,
directory=None,
mounts=None,
isolate=False,
@@ -138,16 +139,118 @@ class Stream():
# in which case we just blindly trust the directory, using the element
# definitions to control the execution environment only.
if directory is None:
- missing_deps = [
- dep._get_full_name()
- for dep in self._pipeline.dependencies([element], scope)
- if not dep._cached()
- ]
+ missing_deps = []
+ for element, scope in elements:
+ missing_deps.extend(
+ dep._get_full_name()
+ for dep in self._pipeline.dependencies((element,), scope)
+ if not dep._cached())
if missing_deps:
raise StreamError("Elements need to be built or downloaded before staging a shell environment",
detail="\n".join(missing_deps))
- return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
+ # Assert we're not mixing virtual directory compatible
+ # and non-virtual directory compatible elements
+ if (any(e.BST_VIRTUAL_DIRECTORY for (e, _) in elements) and
+ not all(e.BST_VIRTUAL_DIRECTORY for (e, _) in elements)):
+ raise StreamError(
+ "Elements do not support multiple-element staging",
+ detail=("Multi-element staging is not supported" +
+ " because elements {} support BST_VIRTUAL_DIRECTORY and {} do not.").format(
+ ', '.join(e.name for (e, _) in elements if e.BST_VIRTUAL_DIRECTORY),
+ ', '.join(e.name for (e, _) in elements if not e.BST_VIRTUAL_DIRECTORY)))
+
+ with ExitStack() as stack:
+ # Creation logic duplicated from Element.__sandbox
+ # since most of it is creating the tmpdir
+ # and deciding whether to make a remote sandbox,
+ # which we don't want to.
+ if directory is None:
+ os.makedirs(self._context.builddir, exist_ok=True)
+ rootdir = stack.enter_context(TemporaryDirectory(dir=self._context.builddir))
+ else:
+ rootdir = directory
+
+ # SandboxConfig comes from project, element defaults and MetaElement sandbox config
+ # In the absence of it being exposed to other APIs and a merging strategy
+ # just make it from the project sandbox config.
+ sandbox_config = SandboxConfig(_yaml.node_get(self._project._sandbox, int, 'build-uid'),
+ _yaml.node_get(self._project._sandbox, int, 'build-gid'))
+ platform = Platform.get_platform()
+ sandbox = platform.create_sandbox(context=self._context,
+ project=self._project,
+ directory=rootdir,
+ stdout=None, stderr=None, config=sandbox_config,
+ bare_directory=directory is not None,
+ allow_real_directory=not any(e.BST_VIRTUAL_DIRECTORY
+ for (e, _) in elements))
+
+ # Configure the sandbox with the last element taking precedence for config.
+ for e, _ in elements:
+ e.configure_sandbox(sandbox)
+
+ # Stage contents if not passed --sysroot
+ if not directory:
+ if any(e.BST_STAGE_INTEGRATES for (e, _) in elements):
+ if len(elements) > 1:
+ raise StreamError(
+ "Elements do not support multiple-element staging",
+ detail=("Elements {} do not support multi-element staging " +
+ " because element kinds {} set BST_STAGE_INTEGRATES").format(
+ ', '.join(e.name for (e, _) in elements if e.BST_STAGE_INTEGRATES),
+ ', '.join(set(e.get_kind() for (e, _) in elements))))
+ elements[0][0].stage(sandbox)
+ else:
+ visited = {}
+ for e, scope in elements:
+ if scope is Scope.BUILD:
+ e.stage(sandbox, visited=visited)
+ else:
+ e.stage_dependency_artifacts(sandbox, scope, visited=visited)
+
+ visited = {}
+ for e, scope in elements:
+ e.integrate_dependency_artifacts(sandbox, scope, visited=visited)
+
+ environment = {}
+ for e, _ in elements:
+ environment.update(e.get_environment())
+ flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
+ shell_command, shell_environment, shell_host_files = self._project.get_shell_config()
+ environment['PS1'] = prompt
+ # Special configurations for non-isolated sandboxes
+ if not isolate:
+
+ # Open the network, and reuse calling uid/gid
+ #
+ flags |= SandboxFlags.NETWORK_ENABLED | SandboxFlags.INHERIT_UID
+
+ # Apply project defined environment vars to set for a shell
+ for key, value in _yaml.node_items(shell_environment):
+ environment[key] = value
+
+ # Setup any requested bind mounts
+ if mounts is None:
+ mounts = []
+
+ for mount in shell_host_files + mounts:
+ if not os.path.exists(mount.host_path):
+ if not mount.optional:
+ self._message(MessageType.WARN,
+ "Not mounting non-existing host file: {}".format(mount.host_path))
+ else:
+ sandbox.mark_directory(mount.path)
+ sandbox._set_mount_source(mount.path, mount.host_path)
+
+ if command:
+ argv = list(command)
+ else:
+ argv = shell_command
+
+ self._message(MessageType.STATUS, "Running command", detail=" ".join(argv))
+
+ # Run shells with network enabled and readonly root.
+ return sandbox.run(argv, flags, env=environment)
# build()
#
[buildstream] 05/12: scriptelement: Skip integration in stage if
BST_STAGE_INTEGRATES not set
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 173adb7da050d580c2020f88eeea84bb3799b85e
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 13:30:54 2018 +0000
scriptelement: Skip integration in stage if BST_STAGE_INTEGRATES not set
This takes the liberty of splitting the layout handling code paths
into separate methods to reduce the amount of nested conditionals
with the aim of improving readability.
---
buildstream/scriptelement.py | 105 ++++++++++++++++++++++++-------------------
1 file changed, 58 insertions(+), 47 deletions(-)
diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py
index 697cd28..0c82b19 100644
--- a/buildstream/scriptelement.py
+++ b/buildstream/scriptelement.py
@@ -216,61 +216,72 @@ class ScriptElement(Element):
if directory != '/':
sandbox.mark_directory(directory, artifact=artifact)
- def stage(self, sandbox):
+ def stage(self, sandbox, *, visited=None):
+ assert not self.BST_STAGE_INTEGRATES or visited is None
# Stage the elements, and run integration commands where appropriate.
- if not self.__layout:
- # if no layout set, stage all dependencies into /
- for build_dep in self.dependencies(Scope.BUILD, recurse=False):
- with self.timed_activity("Staging {} at /"
- .format(build_dep.name), silent_nested=True):
- build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/")
-
- with sandbox.batch(SandboxFlags.NONE):
- for build_dep in self.dependencies(Scope.BUILD, recurse=False):
- with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
- for dep in build_dep.dependencies(Scope.RUN):
- dep.integrate(sandbox)
+ if self.__layout:
+ self.__stage_dependency_artifacts_with_layout(sandbox, visited=visited)
else:
- # If layout, follow its rules.
- for item in self.__layout:
+ self.stage_dependency_artifacts(sandbox, Scope.BUILD, visited=visited)
- # Skip layout members which dont stage an element
- if not item['element']:
- continue
-
- element = self.search(Scope.BUILD, item['element'])
- if item['destination'] == '/':
- with self.timed_activity("Staging {} at /".format(element.name),
- silent_nested=True):
- element.stage_dependency_artifacts(sandbox, Scope.RUN)
- else:
- with self.timed_activity("Staging {} at {}"
- .format(element.name, item['destination']),
- silent_nested=True):
- virtual_dstdir = sandbox.get_virtual_directory()
- virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True)
- element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
-
- with sandbox.batch(SandboxFlags.NONE):
- for item in self.__layout:
-
- # Skip layout members which dont stage an element
- if not item['element']:
- continue
-
- element = self.search(Scope.BUILD, item['element'])
-
- # Integration commands can only be run for elements staged to /
- if item['destination'] == '/':
- with self.timed_activity("Integrating {}".format(element.name),
- silent_nested=True):
- for dep in element.dependencies(Scope.RUN):
- dep.integrate(sandbox)
+ if self.BST_STAGE_INTEGRATES:
+ self.integrate_dependency_artifacts(sandbox, Scope.BUILD)
install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep)
sandbox.get_virtual_directory().descend(install_root_path_components, create=True)
+ def integrate_dependency_artifacts(self, sandbox, scope, *, visited=None):
+ if scope is not Scope.BUILD or not self.__layout:
+ super().integrate_dependency_artifacts(sandbox, scope, visited=visited)
+ return
+
+ with sandbox.batch(SandboxFlags.NONE):
+ self.__integrate_dependency_artifacts_with_layout(sandbox, visited=visited)
+
+ def __stage_dependency_artifacts_with_layout(self, sandbox, *, path=None,
+ include=None, exclude=None, orphans=True, visited=None):
+ if visited is None:
+ visited = {}
+
+ for item in self.__layout:
+
+ # Skip layout members which dont stage an element
+ if not item['element']:
+ continue
+
+ element = self.search(Scope.BUILD, item['element'])
+ if item['destination'] == '/':
+ with self.timed_activity("Staging {} at /".format(element.name), silent_nested=True):
+ element.stage_dependency_artifacts(sandbox, Scope.RUN, path="/", visited=visited)
+ else:
+ path = item['destination']
+ with self.timed_activity("Staging {} at {}".format(element.name, path),
+ silent_nested=True):
+ virtual_dstdir = sandbox.get_virtual_directory()
+ virtual_dstdir.descend(path.lstrip(os.sep).split(os.sep), create=True)
+ # Visited is explicitly not passed here so that staging at an alternative path
+ # doesn't prevent an artifact being staged at root via another dependency.
+ element.stage_dependency_artifacts(sandbox, Scope.RUN, path=path)
+
+ def __integrate_dependency_artifacts_with_layout(self, sandbox, *, visited=None):
+ if visited is None:
+ visited = {}
+
+ for item in self.__layout:
+
+ # Skip layout members which dont stage an element
+ if not item['element']:
+ continue
+
+ element = self.search(Scope.BUILD, item['element'])
+
+ # Integration commands can only be run for elements staged to /
+ if item['destination'] == '/':
+ with self.timed_activity("Integrating {}".format(element.name),
+ silent_nested=True):
+ element.integrate_dependency_artifacts(sandbox, Scope.RUN, visited=visited)
+
def assemble(self, sandbox):
flags = SandboxFlags.NONE
[buildstream] 04/12: element: use BST_STAGE_INTEGRATES in
prepare_sandbox and assemble
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6b9a93fda6a35293c9885ef5d091c6efc23a4bd4
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 13:21:48 2018 +0000
element: use BST_STAGE_INTEGRATES in prepare_sandbox and assemble
This ensures Element.integrate_dependency_artifacts is called after stage
when BST_STAGE_INTEGRATES hasn't been set to False.
---
buildstream/element.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index 13140a1..35c305f 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1395,6 +1395,8 @@ class Element(Plugin):
if not directory:
if shell and scope == Scope.BUILD:
self.stage(sandbox)
+ if not self.BST_STAGE_INTEGRATES:
+ self.integrate_dependency_artifacts(sandbox, scope)
else:
# Stage deps in the sandbox root
with self.timed_activity("Staging dependencies", silent_nested=True):
@@ -1404,8 +1406,7 @@ class Element(Plugin):
# once they are all staged and ready
if integrate:
with self.timed_activity("Integrating sandbox"):
- for dep in self.dependencies(scope):
- dep.integrate(sandbox)
+ self.integrate_dependency_artifacts(sandbox, scope)
yield sandbox
@@ -1624,6 +1625,8 @@ class Element(Plugin):
self.__configure_sandbox(sandbox)
# Step 2 - Stage
self.stage(sandbox)
+ if not self.BST_STAGE_INTEGRATES:
+ self.integrate_dependency_artifacts(sandbox, Scope.BUILD)
if self.__batch_prepare_assemble:
cm = sandbox.batch(self.__batch_prepare_assemble_flags,
[buildstream] 02/12: manual.py: Add BST_VIRTUAL_DIRECTORY flag
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 55259205e6c534c84b172326fef34cdf781415d7
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Wed Nov 7 15:48:42 2018 +0000
manual.py: Add BST_VIRTUAL_DIRECTORY flag
---
buildstream/plugins/elements/manual.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/buildstream/plugins/elements/manual.py b/buildstream/plugins/elements/manual.py
index 3300d01..8f951ab 100644
--- a/buildstream/plugins/elements/manual.py
+++ b/buildstream/plugins/elements/manual.py
@@ -36,7 +36,8 @@ from buildstream import BuildElement
# Element implementation for the 'manual' kind.
class ManualElement(BuildElement):
- pass
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
# Plugin entry point
[buildstream] 11/12: tests: Test multiple-element runtime and
workspace shells
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 87b660fae243cd4ec1e4a8241cde0942fe139d1b
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 16:44:48 2018 +0000
tests: Test multiple-element runtime and workspace shells
---
.../project/elements/shell/adds-bar.bst | 11 ++++
.../project/elements/shell/adds-foo.bst | 11 ++++
tests/integration/shell.py | 62 ++++++++++++++++++++--
3 files changed, 79 insertions(+), 5 deletions(-)
diff --git a/tests/integration/project/elements/shell/adds-bar.bst b/tests/integration/project/elements/shell/adds-bar.bst
new file mode 100644
index 0000000..c2f86bd
--- /dev/null
+++ b/tests/integration/project/elements/shell/adds-bar.bst
@@ -0,0 +1,11 @@
+kind: manual
+depends:
+- base.bst
+
+config:
+ install-commands:
+ - |
+ install -D -m775 /proc/self/fd/0 %{install-root}%{bindir}/bar <<\EOF
+ #!/bin/sh
+ echo bar
+ EOF
diff --git a/tests/integration/project/elements/shell/adds-foo.bst b/tests/integration/project/elements/shell/adds-foo.bst
new file mode 100644
index 0000000..a3d0681
--- /dev/null
+++ b/tests/integration/project/elements/shell/adds-foo.bst
@@ -0,0 +1,11 @@
+kind: manual
+depends:
+- base.bst
+
+config:
+ install-commands:
+ - |
+ install -D -m775 /proc/self/fd/0 %{install-root}%{bindir}/foo <<\EOF
+ #!/bin/sh
+ echo foo
+ EOF
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index d4e672a..a564a68 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -28,11 +28,17 @@ DATA_DIR = os.path.join(
# config (dict): A project.conf dictionary to composite over the default
# mount (tuple): A (host, target) tuple for the `--mount` option
# element (str): The element to build and run a shell with
+# elements (list): Other elements to build and run a shell with
# isolate (bool): Whether to pass --isolate to `bst shell`
#
-def execute_shell(cli, project, command, *, config=None, mount=None, element='base.bst', isolate=False):
+def execute_shell(cli, project, command, *, config=None, mount=None, elements=None, isolate=False):
# Ensure the element is built
- result = cli.run(project=project, project_config=config, args=['build', element])
+ if elements is None:
+ elements = ('base.bst',)
+
+ args = ['build', '--']
+ args.extend(elements)
+ result = cli.run(project=project, project_config=config, args=args)
assert result.exit_code == 0
args = ['shell']
@@ -41,7 +47,9 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element='ba
if mount is not None:
host_path, target_path = mount
args += ['--mount', host_path, target_path]
- args += [element, '--'] + command
+ args.append('--')
+ args.extend(elements)
+ args += ['--'] + command
return cli.run(project=project, project_config=config, args=args)
@@ -158,7 +166,7 @@ def test_no_shell(cli, tmpdir, datafiles):
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
_yaml.dump(element, os.path.join(element_path, element_name))
- result = execute_shell(cli, project, ['/bin/echo', 'Pegasissies!'], element=element_name)
+ result = execute_shell(cli, project, ['/bin/echo', 'Pegasissies!'], elements=(element_name,))
assert result.exit_code == 0
assert result.output == "Pegasissies!\n"
@@ -352,5 +360,49 @@ def test_integration_devices(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'shell/integration.bst'
- result = execute_shell(cli, project, ["true"], element=element_name)
+ result = execute_shell(cli, project, ["true"], elements=(element_name,))
+ assert result.exit_code == 0
+
+
+# Test multiple element shell
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+def test_shell_multiple_elements(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ result = execute_shell(cli, project, ["sh", "-c", "foo && bar"],
+ elements=["shell/adds-foo.bst", "shell/adds-bar.bst"])
+ assert result.exit_code == 0
+
+
+# Test multiple element build shell
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+def test_shell_multiple_workspace(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ elements = {'workspace/workspace-mount.bst': os.path.join(cli.directory, 'workspace-mount'),
+ 'make/makehello.bst': os.path.join(cli.directory, 'makehello')}
+
+ for element, workspace in elements.items():
+ res = cli.run(project=project, args=['workspace', 'open', element, '--directory', workspace])
+ assert res.exit_code == 0
+
+ for workspace in elements.values():
+ with open(os.path.join(workspace, "workspace-exists"), "w") as f:
+ pass
+
+ # Ensure the dependencies of our build failing element are built
+ result = cli.run(project=project, args=['build', 'base.bst', 'make/makehello.bst'])
+ assert result.exit_code == 0
+
+ # Test that only the first workspace exists, since only the first element may be staged for build,
+ # additional elements may only be staged as extra dependencies.
+ args = ['shell', '--build', '--'] + list(elements)
+ args += ['--', 'sh', '-c',
+ 'test -e /buildstream/test/workspace/workspace-mount.bst/workspace-exists && \
+ test ! -e /buildstream/test/make/makehello.bst/workspace-exists']
+ result = cli.run(project=project, args=args)
assert result.exit_code == 0
+ assert result.output == ''
[buildstream] 12/12: Add to NEWS
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/shell-multi-stage
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f7954ef50ee66b9ae1a89155870bc86291b8eac8
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Oct 29 17:46:28 2018 +0000
Add to NEWS
---
NEWS | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/NEWS b/NEWS
index 1b186ce..19154d3 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,12 @@
buildstream 1.3.1
=================
+ o `bst shell` learned to accept multiple elements for staging
+ provided the element's kind flags `BST_STAGE_INTEGRATES` as false.
+
+ As a side-effect it is no longer possible to intersperse options and flags
+ with arguments in the `bst shell` command-line.
+
o All elements must now be suffixed with `.bst`
Attempting to use an element that does not have the `.bst` extension,
will result in a warning.