You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:30:32 UTC
[buildstream] branch except_intersections created (now dd7f8c4)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch except_intersections
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at dd7f8c4 Fix bst-here
This branch includes the following new commits:
new bb99f21 main.py: Fix app initialization
new edb0243 Load excepted elements as part of the pipeline
new 0bec3b6 _pipeline.py: Adjust remove_elements to work on the 'intersection'
new 1650631 Fix tests
new 1a086dc load.py: Add test to check intersection exceptions
new dd7f8c4 Fix bst-here
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 05/06: load.py: Add test to check intersection
exceptions
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch except_intersections
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1a086dc9813d1416a0a54e3dcd75eaa8ef5f9a5d
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Thu Nov 9 14:53:01 2017 +0000
load.py: Add test to check intersection exceptions
---
tests/pipeline/load.py | 20 ++++++++++++++++++++
tests/pipeline/load/remove/unrelated.bst | 4 ++++
2 files changed, 24 insertions(+)
diff --git a/tests/pipeline/load.py b/tests/pipeline/load.py
index eeeeb1d..cc818dc 100644
--- a/tests/pipeline/load.py
+++ b/tests/pipeline/load.py
@@ -130,3 +130,23 @@ def test_remove_elements(cli, datafiles, tmpdir):
assert(element_list[2] == 'first-level-1.bst')
assert(element_list[3] == 'first-level-2.bst')
assert(element_list[4] == 'build.bst')
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'remove'))
+def test_remove_unrelated_element(cli, datafiles, tmpdir):
+ basedir = os.path.join(datafiles.dirname, datafiles.basename)
+ elements = ['build.bst']
+ except_ = ['unrelated.bst']
+
+ # Ensure that we don't just except the given element, but the
+ # first row of intersection elements, while still including things
+ # that are accessible through another route.
+ element_list = cli.get_pipeline(basedir, elements, except_=except_, scope='all')
+
+ assert(len(element_list) == 5)
+
+ assert(element_list[0] == 'fourth-level-2.bst')
+ assert(element_list[1] == 'third-level-2.bst')
+ assert(element_list[2] == 'first-level-1.bst')
+ assert(element_list[3] == 'first-level-2.bst')
+ assert(element_list[4] == 'build.bst')
diff --git a/tests/pipeline/load/remove/unrelated.bst b/tests/pipeline/load/remove/unrelated.bst
new file mode 100644
index 0000000..10de373
--- /dev/null
+++ b/tests/pipeline/load/remove/unrelated.bst
@@ -0,0 +1,4 @@
+kind: autotools
+description: Unrelated to the rest of the pipeline, not loaded when targeting build.bst
+depends:
+ - second-level-1.bst
[buildstream] 03/06: _pipeline.py: Adjust remove_elements to work
on the 'intersection'
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch except_intersections
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0bec3b6828d89c014ba3072173e472237a84d91f
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Nov 7 18:06:41 2017 +0000
_pipeline.py: Adjust remove_elements to work on the 'intersection'
---
buildstream/_pipeline.py | 80 +++++++++++++++++++++++++++++-------------------
1 file changed, 48 insertions(+), 32 deletions(-)
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index f89ad2d..de916f4 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -25,6 +25,7 @@ import stat
import shlex
import shutil
import tarfile
+import itertools
from operator import itemgetter
from tempfile import TemporaryDirectory
from pluginbase import PluginBase
@@ -236,6 +237,8 @@ class Pipeline():
# ordering for optimal build plans
def plan(self):
build_plan = Planner().plan(self.targets)
+ self.remove_elements(build_plan)
+
for element in build_plan:
yield element
@@ -710,48 +713,61 @@ class Pipeline():
#
# Internal function
#
- # Returns all elements to be removed from the given list of
- # elements when the given removed elements and their unique
- # dependencies are removed.
+ # Return what we are left with after the intersection between
+ # excepted and target elements and their unique dependencies is
+ # gone.
#
# Args:
- # elements (list of elements): The graph to sever elements from.
- # removed (list of strings): Names of the elements to remove.
- def remove_elements(self, tree, removed):
+ # elements (list of elements): The list to remove elements from.
+ def remove_elements(self, elements):
+ targeted = list(self.dependencies(Scope.ALL))
+
+ visited = []
- if removed is None:
- removed = []
+ def find_intersection(element):
+ if element in visited:
+ return
+ visited.append(element)
- to_remove = set()
- tree = list(tree)
+ # Intersection elements are those that are also in
+ # 'targeted', as long as we don't recurse into them.
+ if element in targeted:
+ yield element
+ else:
+ for dep in element.dependencies(Scope.ALL, recurse=False):
+ yield from find_intersection(dep)
- # Find all elements that might need to be removed.
- def search_tree(element_name):
- for element in tree:
- if element.name == element_name:
- return element
- return None
+ # Build a list of 'intersection' elements, i.e. the set of
+ # elements that lie on the border closest to excepted elements
+ # between excepted and target elements.
+ intersection = list(itertools.chain.from_iterable(
+ find_intersection(element) for element in self.exceptions
+ ))
- for element_name in removed:
- element = search_tree(element_name)
- if element is None:
- raise PipelineError("No element named {} in the loaded pipeline".format(element_name))
+ # Now use this set of elements to traverse the targeted
+ # elements, except 'intersection' elements and their unique
+ # dependencies.
+ visited = []
- to_remove.update(element.dependencies(Scope.ALL))
+ def traverse(element):
+ if element in visited or element in intersection:
+ return
+ visited.append(element)
- old_to_remove = set()
- while old_to_remove != to_remove:
- old_to_remove = to_remove
+ for dep in element.dependencies(Scope.ALL, recurse=False):
+ traverse(dep)
- # Of these, find all elements that are not a dependency of
- # elements still in use.
- for element in tree:
- if element.name not in removed and element not in to_remove:
- to_remove = to_remove.difference(element.dependencies(Scope.ALL, recurse=False))
+ for element in self.targets:
+ traverse(element)
- to_remove = to_remove.union([e for e in tree if e.name in removed])
+ # That looks like a lot, but overall we only traverse (part
+ # of) the graph twice. This could be reduced to once if we
+ # kept track of parent elements, but is probably not
+ # significant.
- return [element for element in tree if element not in to_remove]
+ # Ensure that we return elements in the same order they were
+ # in before.
+ return [element for element in elements if element in visited]
def validate_workspace_index(self, source_index):
sources = list(self.targets[0].sources())
@@ -787,7 +803,7 @@ class Pipeline():
elements = list(self.dependencies(scope))
- return self.remove_elements(elements, except_)
+ return self.remove_elements(elements)
# source_bundle()
#
[buildstream] 04/06: Fix tests
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch except_intersections
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1650631337552922f6c0bc968c7ae48e3d1c5b11
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Nov 7 18:07:20 2017 +0000
Fix tests
---
tests/pipeline/load.py | 11 -----------
tests/plugins/pipeline.py | 2 +-
tests/project/plugins.py | 2 +-
tests/variables/variables.py | 2 +-
4 files changed, 3 insertions(+), 14 deletions(-)
diff --git a/tests/pipeline/load.py b/tests/pipeline/load.py
index 9cd2fea..eeeeb1d 100644
--- a/tests/pipeline/load.py
+++ b/tests/pipeline/load.py
@@ -14,17 +14,6 @@ DATA_DIR = os.path.join(
)
-def create_pipeline(tmpdir, basedir, target):
- context = Context([])
- project = Project(basedir, context)
-
- context.deploydir = os.path.join(str(tmpdir), 'deploy')
- context.artifactdir = os.path.join(str(tmpdir), 'artifact')
- context._platform = Platform.get_platform()
-
- return Pipeline(context, project, [target])
-
-
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'simple'))
def test_load_simple(cli, datafiles, tmpdir):
basedir = os.path.join(datafiles.dirname, datafiles.basename)
diff --git a/tests/plugins/pipeline.py b/tests/plugins/pipeline.py
index 5f1a85e..8f85253 100644
--- a/tests/plugins/pipeline.py
+++ b/tests/plugins/pipeline.py
@@ -24,7 +24,7 @@ def create_pipeline(tmpdir, basedir, target):
context.artifactdir = os.path.join(str(tmpdir), 'artifact')
context._platform = Platform.get_platform()
- return Pipeline(context, project, [target])
+ return Pipeline(context, project, [target], [])
@pytest.mark.skipif(not HAVE_ROOT, reason="requires root permissions")
diff --git a/tests/project/plugins.py b/tests/project/plugins.py
index 5f75172..ea21730 100644
--- a/tests/project/plugins.py
+++ b/tests/project/plugins.py
@@ -21,7 +21,7 @@ def create_pipeline(tmpdir, basedir, target):
context.artifactdir = os.path.join(str(tmpdir), 'artifact')
context._platform = Platform.get_platform()
- return Pipeline(context, project, [target])
+ return Pipeline(context, project, [target], [])
# We've already validated that the plugin system works in
diff --git a/tests/variables/variables.py b/tests/variables/variables.py
index 7562857..04ac921 100644
--- a/tests/variables/variables.py
+++ b/tests/variables/variables.py
@@ -20,7 +20,7 @@ def create_pipeline(tmpdir, basedir, target):
context.artifactdir = os.path.join(str(tmpdir), 'artifact')
context._platform = Platform.get_platform()
- return Pipeline(context, project, [target])
+ return Pipeline(context, project, [target], [])
def assert_command(datafiles, tmpdir, target, command, expected):
[buildstream] 06/06: Fix bst-here
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch except_intersections
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit dd7f8c43ce81dae3af51af2281174b0894ef7f84
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Wed Nov 15 14:08:42 2017 +0000
Fix bst-here
---
contrib/bst-here | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/contrib/bst-here b/contrib/bst-here
index 6b4e6b2..a88a635 100755
--- a/contrib/bst-here
+++ b/contrib/bst-here
@@ -66,12 +66,16 @@ fi
exec docker run --rm -i${is_tty:+ -t} \
--cap-add SYS_ADMIN \
+ --cap-add SYS_CHROOT \
+ --cap-add NET_ADMIN \
+ --cap-add SETUID \
+ --cap-add SETGID \
--env PS1="$BST_HERE_PS1" \
--device /dev/fuse \
--security-opt seccomp=unconfined \
--volume buildstream-cache:/root/.cache/buildstream \
--volume buildstream-config:/root/.config \
- --volume "$PWD":/src \
+ --volume "$PWD":/src:Z \
--workdir /src \
buildstream/buildstream-fedora:latest \
$command
[buildstream] 01/06: main.py: Fix app initialization
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch except_intersections
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bb99f2110fbf6a9788c7e4517ca2c7e2ad9fcbd4
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Nov 7 18:02:07 2017 +0000
main.py: Fix app initialization
---
buildstream/_frontend/main.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 4e00a2b..3189e10 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -474,7 +474,7 @@ def shell(app, element, sysroot, build, command):
else:
scope = Scope.RUN
- app.initialize([element])
+ app.initialize((element,))
# Assert we have everything we need built.
missing_deps = []
@@ -516,7 +516,7 @@ def shell(app, element, sysroot, build, command):
def checkout(app, element, directory, force, integrate):
"""Checkout a built artifact to the specified directory
"""
- app.initialize([element])
+ app.initialize((element,))
try:
app.pipeline.checkout(directory, force, integrate)
click.echo("")
@@ -548,7 +548,7 @@ def checkout(app, element, directory, force, integrate):
def source_bundle(app, target, force, directory,
track, compression, except_):
"""Produce a source bundle to be manually executed"""
- app.initialize([target], rewritable=track, inconsistent=track)
+ app.initialize((target,), rewritable=track, inconsistent=track)
try:
dependencies = app.pipeline.deps_elements('all', except_)
app.print_heading(dependencies)
@@ -589,7 +589,7 @@ def workspace():
def workspace_open(app, no_checkout, force, source, track, element, directory):
"""Open a workspace for manual source modification"""
- app.initialize([element], rewritable=track, inconsistent=track)
+ app.initialize((element,), rewritable=track, inconsistent=track)
try:
app.pipeline.open_workspace(app.scheduler, directory, source, no_checkout, track, force)
click.echo("")
@@ -613,7 +613,7 @@ def workspace_open(app, no_checkout, force, source, track, element, directory):
def workspace_close(app, source, remove_dir, element):
"""Close a workspace"""
- app.initialize([element])
+ app.initialize((element,))
if app.interactive and remove_dir:
if not click.confirm('This will remove all your changes, are you sure?'):
click.echo('Aborting')
@@ -643,7 +643,7 @@ def workspace_close(app, source, remove_dir, element):
@click.pass_obj
def workspace_reset(app, source, track, no_checkout, element):
"""Reset a workspace to its original state"""
- app.initialize([element])
+ app.initialize((element,))
if app.interactive:
if not click.confirm('This will remove all your changes, are you sure?'):
click.echo('Aborting')
[buildstream] 02/06: Load excepted elements as part of the pipeline
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch except_intersections
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit edb0243475ac854c97b57c4d35e7227483f2b1a3
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Nov 7 18:04:13 2017 +0000
Load excepted elements as part of the pipeline
---
buildstream/_frontend/main.py | 21 ++++++++++++---------
buildstream/_pipeline.py | 12 ++++++++----
2 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 3189e10..446445c 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -242,9 +242,10 @@ def fetch(app, elements, deps, track, except_):
plan: Only dependencies required for the build plan
all: All dependencies
"""
- app.initialize(elements, rewritable=track, inconsistent=track)
+ app.initialize(elements, except_=except_,
+ rewritable=track, inconsistent=track)
try:
- dependencies = app.pipeline.deps_elements(deps, except_)
+ dependencies = app.pipeline.deps_elements(deps)
app.print_heading(deps=dependencies)
app.pipeline.fetch(app.scheduler, dependencies, track)
click.echo("")
@@ -281,9 +282,10 @@ def track(app, elements, deps, except_):
none: No dependencies, just the element itself
all: All dependencies
"""
- app.initialize(elements, rewritable=True, inconsistent=True)
+ app.initialize(elements, except_=except_,
+ rewritable=True, inconsistent=True)
try:
- dependencies = app.pipeline.deps_elements(deps, except_)
+ dependencies = app.pipeline.deps_elements(deps)
app.print_heading(deps=dependencies)
app.pipeline.track(app.scheduler, dependencies)
click.echo("")
@@ -422,9 +424,9 @@ def show(app, elements, deps, except_, order, format, downloadable):
bst show target.bst --format \\
$'---------- %{name} ----------\\n%{vars}'
"""
- app.initialize(elements, fetch_remote_refs=downloadable)
+ app.initialize(elements, except_=except_, fetch_remote_refs=downloadable)
try:
- dependencies = app.pipeline.deps_elements(deps, except_)
+ dependencies = app.pipeline.deps_elements(deps)
except PipelineError as e:
click.echo("{}".format(e))
sys.exit(-1)
@@ -550,7 +552,7 @@ def source_bundle(app, target, force, directory,
"""Produce a source bundle to be manually executed"""
app.initialize((target,), rewritable=track, inconsistent=track)
try:
- dependencies = app.pipeline.deps_elements('all', except_)
+ dependencies = app.pipeline.deps_elements('all')
app.print_heading(dependencies)
app.pipeline.source_bundle(app.scheduler, dependencies, force, track,
compression, directory)
@@ -762,7 +764,8 @@ class App():
#
# Initialize the main pipeline
#
- def initialize(self, elements, rewritable=False, inconsistent=False, fetch_remote_refs=False):
+ def initialize(self, elements, except_=tuple(), rewritable=False,
+ inconsistent=False, fetch_remote_refs=False):
profile_start(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, '-') for t in elements))
@@ -835,7 +838,7 @@ class App():
sys.exit(-1)
try:
- self.pipeline = Pipeline(self.context, self.project, elements,
+ self.pipeline = Pipeline(self.context, self.project, elements, except_,
inconsistent=inconsistent,
rewritable=rewritable,
fetch_remote_refs=fetch_remote_refs,
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 9860d29..f89ad2d 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -112,7 +112,7 @@ class Planner():
#
class Pipeline():
- def __init__(self, context, project, targets,
+ def __init__(self, context, project, targets, except_,
inconsistent=False,
rewritable=False,
fetch_remote_refs=False,
@@ -127,7 +127,7 @@ class Pipeline():
self.unused_workspaces = []
self._resolved_elements = {}
- loader = Loader(self.project.element_path, targets,
+ loader = Loader(self.project.element_path, targets + except_,
self.project._options)
meta_elements = loader.load(rewritable, load_ticker)
if load_ticker:
@@ -144,8 +144,12 @@ class Pipeline():
self.source_factory = SourceFactory(pluginbase, project._plugin_source_paths)
# Resolve the real elements now that we've resolved the project
- self.targets = [self.resolve(meta_element, ticker=resolve_ticker)
- for meta_element in meta_elements]
+ resolved_elements = [self.resolve(meta_element, ticker=resolve_ticker)
+ for meta_element in meta_elements]
+
+ self.targets = resolved_elements[:len(targets)]
+ self.exceptions = resolved_elements[len(targets):]
+
if resolve_ticker:
resolve_ticker(None)