You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:42:40 UTC
[buildstream] branch tracking-changes created (now 14ef3ba)
This is an automated email from the ASF dual-hosted git repository.
root pushed a change to branch tracking-changes
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 14ef3ba Fix tests
This branch includes the following new commits:
new e4b6fd0 Issue #113: Split tracking and saving in `bst build`
new 4580e2e Issue #117: Add `bst build --track`
new 7513333 Add `bst build --track-except`
new c382086 main.py: Add `bst build --track-all`
new 14ef3ba Fix tests
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 05/05: Fix tests
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tracking-changes
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 14ef3ba4ff703cfe5b524ac5d191cc0a05517ce1
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Fri Oct 20 18:16:07 2017 +0100
Fix tests
---
tests/completions/completions.py | 3 +-
tests/frontend/buildtrack.py | 312 ++++++++++++++++++++++
tests/pipeline/load.py | 4 +
tests/pipeline/load/exceptions/fourth-level-3.bst | 2 +
tests/pipeline/load/exceptions/second-level-1.bst | 3 +-
tests/pipeline/load/exceptions/third-level-3.bst | 4 +
tests/testutils/repo/git.py | 5 +
tests/testutils/runcli.py | 26 ++
8 files changed, 357 insertions(+), 2 deletions(-)
diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index d2c6311..8ecf83e 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -97,7 +97,8 @@ def test_commands(cli, cmd, word_idx, expected):
('bst --l', 1, ['--log-file ']),
# Test that options of subcommands also complete
- ('bst --no-colors build -', 3, ['--all ', '--track ']),
+ ('bst --no-colors build -', 3, ['--all ', '--track ', '--track-all ',
+ '--track-except ', '--track-save ']),
# Test the behavior of completing after an option that has a
# parameter that cannot be completed, vs an option that has
diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py
new file mode 100644
index 0000000..e91274a
--- /dev/null
+++ b/tests/frontend/buildtrack.py
@@ -0,0 +1,312 @@
+import os
+import re
+import shutil
+import itertools
+
+import pytest
+from tests.testutils import cli, create_repo
+
+from buildstream import _yaml
+from buildstream._exceptions import LoadError
+
+
+# Project directory
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ "project",
+)
+
+
+def create_element(repo, name, path, dependencies, ref=None):
+ element = {
+ 'kind': 'import',
+ 'sources': [
+ repo.source_config(ref=ref)
+ ],
+ 'depends': dependencies
+ }
+ _yaml.dump(element, os.path.join(path, name))
+
+
+@pytest.mark.parametrize("save", [([True]), ([False])])
+@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize("exceptions,excepted", [
+ # Test with no exceptions
+ ([], []),
+
+ # Test excepting '2.bst'
+ (['2.bst'], ['2.bst', '7.bst']),
+
+ # Test excepting '2.bst' and '3.bst'
+ (['2.bst', '3.bst'], [
+ '2.bst', '3.bst', '4.bst',
+ '5.bst', '6.bst', '7.bst'
+ ])
+])
+@pytest.mark.parametrize("track_targets,tracked", [
+ # Test tracking the main target element
+ (['0.bst'], [
+ '0.bst', '2.bst', '3.bst',
+ '4.bst', '5.bst', '6.bst', '7.bst'
+ ]),
+
+ # Test tracking a child element
+ (['3.bst'], [
+ '3.bst', '4.bst', '5.bst',
+ '6.bst'
+ ]),
+
+ # Test tracking multiple children
+ (['2.bst', '3.bst'], [
+ '2.bst', '3.bst', '4.bst',
+ '5.bst', '6.bst', '7.bst'
+ ])
+])
+def test_build_track(cli, datafiles, tmpdir, track_targets,
+ exceptions, tracked, excepted, save):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ dev_files_path = os.path.join(project, 'files', 'dev-files')
+ element_path = os.path.join(project, 'elements')
+
+ repo = create_repo('git', str(tmpdir))
+ ref = repo.create(dev_files_path)
+
+ create_elements = {
+ '0.bst': [
+ '2.bst',
+ '3.bst'
+ ],
+ '2.bst': [
+ '3.bst',
+ '7.bst'
+ ],
+ '3.bst': [
+ '4.bst',
+ '5.bst',
+ '6.bst'
+ ],
+ '4.bst': [],
+ '5.bst': [],
+ '6.bst': [
+ '5.bst'
+ ],
+ '7.bst': []
+ }
+ for element, dependencies in create_elements.items():
+ # Test the element inconsistency resolution by ensuring that
+ # only elements that aren't tracked have refs
+ if element in set(tracked) - set(excepted):
+ create_element(repo, element, element_path, dependencies)
+ else:
+ create_element(repo, element, element_path, dependencies, ref=ref)
+
+ args = ['build']
+ if save:
+ args += ['--track-save']
+ args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
+ args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
+ args += ['0.bst']
+
+ result = cli.run(project=project, silent=True, args=args)
+ tracked_elements = result.get_tracked_elements()
+
+ assert set(tracked_elements) == set(tracked) - set(excepted)
+
+ for target in set(tracked) - set(excepted):
+ cli.remove_artifact_from_cache(project, target)
+
+ # Delete element sources
+ source_dir = os.path.join(project, 'cache', 'sources')
+ shutil.rmtree(source_dir)
+
+ if not save:
+ assert cli.get_element_state(project, target) == 'no reference'
+ else:
+ assert cli.get_element_state(project, target) == 'fetch needed'
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize("exceptions,excepted", [
+ # Test with no exceptions
+ ([], []),
+
+ # Test excepting '2.bst'
+ (['2.bst'], ['2.bst', '7.bst']),
+
+ # Test excepting '2.bst' and '3.bst'
+ (['2.bst', '3.bst'], [
+ '2.bst', '3.bst', '4.bst',
+ '5.bst', '6.bst', '7.bst'
+ ])
+])
+@pytest.mark.parametrize("track_targets,tracked", [
+ # Test tracking the main target element
+ (['0.bst'], [
+ '0.bst', '2.bst', '3.bst',
+ '4.bst', '5.bst', '6.bst', '7.bst'
+ ]),
+
+ # Test tracking a child element
+ (['3.bst'], [
+ '3.bst', '4.bst', '5.bst',
+ '6.bst'
+ ]),
+
+ # Test tracking multiple children
+ (['2.bst', '3.bst'], [
+ '2.bst', '3.bst', '4.bst',
+ '5.bst', '6.bst', '7.bst'
+ ])
+])
+def test_build_track_update(cli, datafiles, tmpdir, track_targets,
+ exceptions, tracked, excepted):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ dev_files_path = os.path.join(project, 'files', 'dev-files')
+ element_path = os.path.join(project, 'elements')
+
+ repo = create_repo('git', str(tmpdir))
+ ref = repo.create(dev_files_path)
+
+ create_elements = {
+ '0.bst': [
+ '2.bst',
+ '3.bst'
+ ],
+ '2.bst': [
+ '3.bst',
+ '7.bst'
+ ],
+ '3.bst': [
+ '4.bst',
+ '5.bst',
+ '6.bst'
+ ],
+ '4.bst': [],
+ '5.bst': [],
+ '6.bst': [
+ '5.bst'
+ ],
+ '7.bst': []
+ }
+ for element, dependencies in create_elements.items():
+ # We set a ref for all elements, so that we ensure that we
+ # only track exactly those elements that we want to track,
+ # even if others can be tracked
+ create_element(repo, element, element_path, dependencies, ref=ref)
+ repo.add_commit()
+
+ args = ['build']
+ args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
+ args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
+ args += ['0.bst']
+
+ result = cli.run(project=project, silent=True, args=args)
+ tracked_elements = result.get_tracked_elements()
+
+ assert set(tracked_elements) == set(tracked) - set(excepted)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize("track_targets,exceptions", [
+ # Test tracking the main target element, but excepting some of its
+ # children
+ (['0.bst'], ['6.bst']),
+
+ # Test only tracking a child element
+ (['3.bst'], []),
+])
+def test_build_track_inconsistent(cli, datafiles, tmpdir,
+ track_targets, exceptions):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ dev_files_path = os.path.join(project, 'files', 'dev-files')
+ element_path = os.path.join(project, 'elements')
+
+ repo = create_repo('git', str(tmpdir))
+ repo.create(dev_files_path)
+
+ create_elements = {
+ '0.bst': [
+ '2.bst',
+ '3.bst'
+ ],
+ '2.bst': [
+ '3.bst',
+ '7.bst'
+ ],
+ '3.bst': [
+ '4.bst',
+ '5.bst',
+ '6.bst'
+ ],
+ '4.bst': [],
+ '5.bst': [],
+ '6.bst': [
+ '5.bst'
+ ],
+ '7.bst': []
+ }
+ for element, dependencies in create_elements.items():
+ # We don't add refs so that all elements *have* to be tracked
+ create_element(repo, element, element_path, dependencies)
+
+ args = ['build']
+ args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
+ args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
+ args += ['0.bst']
+
+ result = cli.run(args=args, silent=True)
+
+ assert result.exit_code != 0
+ assert isinstance(result.exception, LoadError)
+
+
+# Assert that if a build element has a dependency in the tracking
+# queue it does not start building before tracking finishes.
+@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize("strict", ['--strict', '--no-strict'])
+def test_build_track_track_first(cli, datafiles, tmpdir, strict):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ dev_files_path = os.path.join(project, 'files', 'dev-files')
+ element_path = os.path.join(project, 'elements')
+
+ repo = create_repo('git', str(tmpdir))
+ ref = repo.create(dev_files_path)
+
+ create_elements = {
+ '0.bst': [
+ '1.bst'
+ ],
+ '1.bst': [],
+ '2.bst': [
+ '0.bst'
+ ]
+ }
+ for element, dependencies in create_elements.items():
+ # We set a ref so that 0.bst can already be built even if
+ # 1.bst has not been tracked yet.
+ create_element(repo, element, element_path, dependencies, ref=ref)
+ repo.add_commit()
+
+ # Build 1.bst and 2.bst first so we have an artifact for them
+ args = [strict, 'build', '2.bst']
+ result = cli.run(args=args, project=project, silent=True)
+ assert result.exit_code == 0
+
+ # Test building 0.bst while tracking 1.bst
+ cli.remove_artifact_from_cache(project, '0.bst')
+
+ args = [strict, 'build', '--track', '1.bst', '2.bst']
+ result = cli.run(args=args, project=project, silent=True)
+ assert result.exit_code == 0
+
+ # Assert that 1.bst successfully tracks before 0.bst builds
+ track_messages = re.finditer(r'\[track:1.bst\s*]', result.output)
+ build_0 = re.search(r'\[build:0.bst\s*] START', result.output).start()
+ assert all(track_message.start() < build_0 for track_message in track_messages)
+
+ # Assert that 2.bst is *only* rebuilt if we are in strict mode
+ build_2 = re.search(r'\[build:2.bst\s*] START', result.output)
+ if strict == '--strict':
+ assert build_2 is not None
+ else:
+ assert build_2 is None
diff --git a/tests/pipeline/load.py b/tests/pipeline/load.py
index 80a366a..fa70fad 100644
--- a/tests/pipeline/load.py
+++ b/tests/pipeline/load.py
@@ -126,6 +126,8 @@ def test_iterate_no_recurse(cli, datafiles, tmpdir):
'third-level-1.bst',
'fourth-level-2.bst',
'third-level-2.bst',
+ 'fourth-level-3.bst',
+ 'third-level-3.bst',
'second-level-1.bst',
'first-level-1.bst',
'first-level-2.bst',
@@ -137,6 +139,8 @@ def test_iterate_no_recurse(cli, datafiles, tmpdir):
(['build.bst'], ['third-level-1.bst'], [
'fourth-level-2.bst',
'third-level-2.bst',
+ 'fourth-level-3.bst',
+ 'third-level-3.bst',
'second-level-1.bst',
'first-level-1.bst',
'first-level-2.bst',
diff --git a/tests/pipeline/load/exceptions/fourth-level-3.bst b/tests/pipeline/load/exceptions/fourth-level-3.bst
new file mode 100644
index 0000000..6812e2f
--- /dev/null
+++ b/tests/pipeline/load/exceptions/fourth-level-3.bst
@@ -0,0 +1,2 @@
+kind: autotools
+description: Should not be removed
diff --git a/tests/pipeline/load/exceptions/second-level-1.bst b/tests/pipeline/load/exceptions/second-level-1.bst
index bd45ef3..3183e67 100644
--- a/tests/pipeline/load/exceptions/second-level-1.bst
+++ b/tests/pipeline/load/exceptions/second-level-1.bst
@@ -1,5 +1,6 @@
kind: autotools
-description: Depends uniquely on one dependency, shares another
+description: Depends uniquely on one dependency, shares another, has another unique nested dependency
depends:
- third-level-1.bst
- third-level-2.bst
+ - third-level-3.bst
diff --git a/tests/pipeline/load/exceptions/third-level-3.bst b/tests/pipeline/load/exceptions/third-level-3.bst
new file mode 100644
index 0000000..56fbe55
--- /dev/null
+++ b/tests/pipeline/load/exceptions/third-level-3.bst
@@ -0,0 +1,4 @@
+kind: autotools
+description: Should be an explicit dependency, and *not* remove its children
+depends:
+ - fourth-level-3.bst
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index 46338f4..57a441a 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -32,6 +32,11 @@ class Git(Repo):
subprocess.call(['git', 'commit', '-m', 'Initial commit'], env=GIT_ENV, cwd=self.repo)
return self.latest_commit()
+ def add_commit(self):
+ subprocess.call(['git', 'commit', '--allow-empty', '-m', 'Additional commit'],
+ env=GIT_ENV, cwd=self.repo)
+ return self.latest_commit()
+
def add_submodule(self, subdir, url):
self.submodules[subdir] = url
subprocess.call(['git', 'submodule', 'add', url, subdir], env=GIT_ENV, cwd=self.repo)
diff --git a/tests/testutils/runcli.py b/tests/testutils/runcli.py
index f165232..202766f 100644
--- a/tests/testutils/runcli.py
+++ b/tests/testutils/runcli.py
@@ -1,5 +1,7 @@
import os
+import re
import sys
+import shutil
import itertools
import traceback
from contextlib import contextmanager, ExitStack
@@ -7,6 +9,8 @@ from click.testing import CliRunner
from ruamel import yaml
import pytest
+from tests.testutils.site import IS_LINUX
+
# Import the main cli entrypoint
from buildstream._frontend.main import cli as bst_cli
from buildstream import _yaml
@@ -24,6 +28,17 @@ class Result():
self.exception = _get_last_exception()
self.result = result
+ ##################################################################
+ # Result parsers #
+ ##################################################################
+ def get_tracked_elements(self):
+ tracked = re.findall(r'\[track:(\S+)\s*]',
+ self.result.output)
+ if tracked is None:
+ return []
+
+ return list(tracked)
+
class Cli():
@@ -44,6 +59,17 @@ class Cli():
def configure(self, config):
self.config = config
+ def remove_artifact_from_cache(self, project, element_name):
+ cache_dir = os.path.join(project, 'cache', 'artifacts')
+
+ if IS_LINUX:
+ cache_dir = os.path.join(cache_dir, 'ostree', 'refs', 'heads')
+ else:
+ cache_dir = os.path.join(cache_dir, 'tar')
+
+ cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
+ shutil.rmtree(cache_dir)
+
# run():
#
# Runs buildstream with the given arguments, additionally
[buildstream] 03/05: Add `bst build --track-except`
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tracking-changes
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 75133339fa327529e10d16dc3191d9eaa4209651
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Oct 24 16:47:35 2017 +0100
Add `bst build --track-except`
---
buildstream/_frontend/main.py | 10 +++++++---
buildstream/_pipeline.py | 2 +-
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index d694d8c..8c47539 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -100,7 +100,8 @@ def override_completions(cmd_param, ctx, args, incomplete):
(cmd_param.name == 'elements' or
cmd_param.name == 'element' or
cmd_param.name == 'except_' or
- cmd_param.opts == ['--track']):
+ cmd_param.opts == ['--track'] or
+ cmd_param.opts == ['--track-except']):
return complete_target(ctx, args, incomplete)
raise CompleteUnhandled()
@@ -198,15 +199,18 @@ def cli(context, **kwargs):
type=click.Path(dir_okay=False, readable=True),
help="Specify elements to track during the build. Can be used "
"repeatedly to specify multiple elements")
+@click.option('--track-except', multiple=True,
+ type=click.Path(dir_okay=False, readable=True),
+ help="Except certain dependencies from tracking")
@click.option('--track-save', default=False, is_flag=True,
help="Write out the tracked references to their element files")
@click.argument('elements', nargs=-1,
type=click.Path(dir_okay=False, readable=True))
@click.pass_obj
-def build(app, elements, all, track, track_save):
+def build(app, elements, all, track, track_save, track_except):
"""Build elements in a pipeline"""
- app.initialize(elements, rewritable=track_save)
+ app.initialize(elements, except_=track_except, rewritable=track_save)
app.pipeline.initialize(use_remote_cache=True, inconsistent=track)
app.print_heading()
try:
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index bd8fbb2..08684d4 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -437,7 +437,7 @@ class Pipeline():
if build_all:
plan = self.dependencies(Scope.ALL)
else:
- plan = self.plan()
+ plan = self.plan(except_=False)
# We want to start the build queue with any elements that are
# not being tracked first
[buildstream] 01/05: Issue #113: Split tracking and saving in `bst
build`
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tracking-changes
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e4b6fd09a47c82d8af5f247116a2a886b8f76c60
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Fri Oct 20 15:07:01 2017 +0100
Issue #113: Split tracking and saving in `bst build`
---
buildstream/_frontend/main.py | 11 ++++++++---
buildstream/_pipeline.py | 9 ++++++---
buildstream/_scheduler/trackqueue.py | 17 ++++++++++-------
3 files changed, 24 insertions(+), 13 deletions(-)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index d706458..8613e1f 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -195,16 +195,21 @@ def cli(context, **kwargs):
help="Build elements that would not be needed for the current build plan")
@click.option('--track', default=False, is_flag=True,
help="Track new source references before building (implies --all)")
+@click.option('--track-save', default=False, is_flag=True,
+ help="Track new source references before building, updating their "
+ "corresponding element files")
@click.argument('elements', nargs=-1,
type=click.Path(dir_okay=False, readable=True))
@click.pass_obj
-def build(app, elements, all, track):
+def build(app, elements, all, track, track_save):
"""Build elements in a pipeline"""
- app.initialize(elements, rewritable=track, inconsistent=track, use_remote_cache=True)
+ track_first = track or track_save
+
+ app.initialize(elements, rewritable=track_save, inconsistent=track_first, use_remote_cache=True)
app.print_heading()
try:
- app.pipeline.build(app.scheduler, all, track)
+ app.pipeline.build(app.scheduler, all, track_first, track_save)
click.echo("")
app.print_summary()
except PipelineError:
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 2a64e2f..fa7e8dc 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -384,9 +384,12 @@ class Pipeline():
# scheduler (Scheduler): The scheduler to run this pipeline on
# build_all (bool): Whether to build all elements, or only those
# which are required to build the target.
- # track_first (bool): Track sources before fetching and building (implies build_all)
+ # track_first (list): Elements whose sources to track prior to
+ # building
+ # save (bool): Whether to save the tracking results in the
+ # elements
#
- def build(self, scheduler, build_all, track_first):
+ def build(self, scheduler, build_all, track_first, save):
if len(self.unused_workspaces) > 0:
self.message(MessageType.WARN, "Unused workspaces",
detail="\n".join([el + "-" + str(src) for el, src, _
@@ -409,7 +412,7 @@ class Pipeline():
push = None
queues = []
if track_first:
- track = TrackQueue()
+ track = TrackQueue(save=save)
queues.append(track)
if self.artifacts.can_fetch():
pull = PullQueue()
diff --git a/buildstream/_scheduler/trackqueue.py b/buildstream/_scheduler/trackqueue.py
index df73a72..8c0ffdc 100644
--- a/buildstream/_scheduler/trackqueue.py
+++ b/buildstream/_scheduler/trackqueue.py
@@ -39,8 +39,9 @@ class TrackQueue(Queue):
complete_name = "Tracked"
queue_type = QueueType.FETCH
- def __init__(self):
+ def __init__(self, save=True):
super(TrackQueue, self).__init__()
+ self.save = save
def process(self, element):
return element._track()
@@ -69,12 +70,14 @@ class TrackQueue(Queue):
# Here we are in master process, what to do if writing
# to the disk fails for some reason ?
- try:
- _yaml.dump(toplevel, fullname)
- except OSError as e:
- source.error("Failed to update project file",
- detail="{}: Failed to rewrite tracked source to file {}: {}"
- .format(source, fullname, e))
+ if self.save:
+ try:
+ _yaml.dump(toplevel, fullname)
+ except OSError as e:
+ source.error("Failed to update project file",
+ detail="{}: Failed to rewrite "
+ "tracked source to file {}: {}"
+ .format(source, fullname, e))
# Forcefully recalculate the element's consistency state after successfully
# tracking, this is avoid a following fetch queue operating on the sources
[buildstream] 04/05: main.py: Add `bst build --track-all`
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tracking-changes
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c38208606ea52c9fb160bec51b145f2080af491c
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Thu Dec 7 11:40:59 2017 +0000
main.py: Add `bst build --track-all`
---
buildstream/_frontend/main.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 8c47539..813d09a 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -202,14 +202,19 @@ def cli(context, **kwargs):
@click.option('--track-except', multiple=True,
type=click.Path(dir_okay=False, readable=True),
help="Except certain dependencies from tracking")
+@click.option('--track-all', default=False, is_flag=True,
+ help="Track all elements in the build pipeline before building")
@click.option('--track-save', default=False, is_flag=True,
help="Write out the tracked references to their element files")
@click.argument('elements', nargs=-1,
type=click.Path(dir_okay=False, readable=True))
@click.pass_obj
-def build(app, elements, all, track, track_save, track_except):
+def build(app, elements, all, track, track_save, track_all, track_except):
"""Build elements in a pipeline"""
+ if track_all:
+ track = elements
+
app.initialize(elements, except_=track_except, rewritable=track_save)
app.pipeline.initialize(use_remote_cache=True, inconsistent=track)
app.print_heading()
[buildstream] 02/05: Issue #117: Add `bst build --track`
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tracking-changes
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4580e2e57e27511366dcd6d6cb10a9b64cf437b2
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Fri Oct 20 16:27:18 2017 +0100
Issue #117: Add `bst build --track`
---
buildstream/_frontend/main.py | 49 +++++++++------
buildstream/_pipeline.py | 138 ++++++++++++++++++++++++++++--------------
2 files changed, 122 insertions(+), 65 deletions(-)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 8613e1f..d694d8c 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -99,7 +99,8 @@ def override_completions(cmd_param, ctx, args, incomplete):
if isinstance(cmd_param.type, click.Path) and \
(cmd_param.name == 'elements' or
cmd_param.name == 'element' or
- cmd_param.name == 'except_'):
+ cmd_param.name == 'except_' or
+ cmd_param.opts == ['--track']):
return complete_target(ctx, args, incomplete)
raise CompleteUnhandled()
@@ -193,23 +194,23 @@ def cli(context, **kwargs):
@cli.command(short_help="Build elements in a pipeline")
@click.option('--all', default=False, is_flag=True,
help="Build elements that would not be needed for the current build plan")
-@click.option('--track', default=False, is_flag=True,
- help="Track new source references before building (implies --all)")
+@click.option('--track', multiple=True,
+ type=click.Path(dir_okay=False, readable=True),
+ help="Specify elements to track during the build. Can be used "
+ "repeatedly to specify multiple elements")
@click.option('--track-save', default=False, is_flag=True,
- help="Track new source references before building, updating their "
- "corresponding element files")
+ help="Write out the tracked references to their element files")
@click.argument('elements', nargs=-1,
type=click.Path(dir_okay=False, readable=True))
@click.pass_obj
def build(app, elements, all, track, track_save):
"""Build elements in a pipeline"""
- track_first = track or track_save
-
- app.initialize(elements, rewritable=track_save, inconsistent=track_first, use_remote_cache=True)
+ app.initialize(elements, rewritable=track_save)
+ app.pipeline.initialize(use_remote_cache=True, inconsistent=track)
app.print_heading()
try:
- app.pipeline.build(app.scheduler, all, track_first, track_save)
+ app.pipeline.build(app.scheduler, all, track, track_save)
click.echo("")
app.print_summary()
except PipelineError:
@@ -248,8 +249,9 @@ def fetch(app, elements, deps, track, except_):
plan: Only dependencies required for the build plan
all: All dependencies
"""
- app.initialize(elements, except_=except_,
- rewritable=track, inconsistent=track)
+
+ app.initialize(elements, except_=except_, rewritable=track)
+ app.pipeline.initialize(inconsistent=elements if track else None)
try:
dependencies = app.pipeline.deps_elements(deps)
app.print_heading(deps=dependencies)
@@ -288,8 +290,8 @@ def track(app, elements, deps, except_):
none: No dependencies, just the element itself
all: All dependencies
"""
- app.initialize(elements, except_=except_,
- rewritable=True, inconsistent=True)
+ app.initialize(elements, except_=except_, rewritable=True)
+ app.pipeline.initialize(inconsistent=elements)
try:
dependencies = app.pipeline.deps_elements(deps)
app.print_heading(deps=dependencies)
@@ -321,7 +323,8 @@ def pull(app, elements, deps):
none: No dependencies, just the element itself
all: All dependencies
"""
- app.initialize(elements, use_remote_cache=True)
+ app.initialize(elements)
+ app.pipeline.initialize(use_remote_cache=True)
try:
to_pull = app.pipeline.deps_elements(deps)
app.pipeline.pull(app.scheduler, to_pull)
@@ -351,7 +354,8 @@ def push(app, elements, deps):
none: No dependencies, just the element itself
all: All dependencies
"""
- app.initialize(elements, use_remote_cache=True)
+ app.initialize(elements)
+ app.pipeline.initialize(use_remote_cache=True)
try:
to_push = app.pipeline.deps_elements(deps)
app.pipeline.push(app.scheduler, to_push)
@@ -430,7 +434,8 @@ def show(app, elements, deps, except_, order, format, downloadable):
bst show target.bst --format \\
$'---------- %{name} ----------\\n%{vars}'
"""
- app.initialize(elements, except_=except_, use_remote_cache=downloadable)
+ app.initialize(elements, except_=except_)
+ app.pipeline.initialize(use_remote_cache=downloadable)
try:
dependencies = app.pipeline.deps_elements(deps)
except PipelineError as e:
@@ -483,6 +488,7 @@ def shell(app, element, sysroot, build, command):
scope = Scope.RUN
app.initialize((element,))
+ app.pipeline.initialize()
# Assert we have everything we need built.
missing_deps = []
@@ -525,6 +531,7 @@ def checkout(app, element, directory, force, integrate):
"""Checkout a built artifact to the specified directory
"""
app.initialize((element,))
+ app.pipeline.initialize()
try:
app.pipeline.checkout(directory, force, integrate)
click.echo("")
@@ -556,7 +563,8 @@ def checkout(app, element, directory, force, integrate):
def source_bundle(app, target, force, directory,
track, compression, except_):
"""Produce a source bundle to be manually executed"""
- app.initialize((target,), rewritable=track, inconsistent=track)
+ app.initialize((target,), rewritable=track)
+ app.pipeline.initialize(inconsistent=[target])
try:
dependencies = app.pipeline.deps_elements('all')
app.print_heading(dependencies)
@@ -597,7 +605,8 @@ def workspace():
def workspace_open(app, no_checkout, force, source, track, element, directory):
"""Open a workspace for manual source modification"""
- app.initialize((element,), rewritable=track, inconsistent=track)
+ app.initialize((element,), rewritable=track)
+ app.pipeline.initialize(inconsistent=[element])
try:
app.pipeline.open_workspace(app.scheduler, directory, source, no_checkout, track, force)
click.echo("")
@@ -622,6 +631,7 @@ def workspace_close(app, source, remove_dir, element):
"""Close a workspace"""
app.initialize((element,))
+ app.pipeline.initialize()
if app.interactive and remove_dir:
if not click.confirm('This will remove all your changes, are you sure?'):
click.echo('Aborting')
@@ -652,6 +662,7 @@ def workspace_close(app, source, remove_dir, element):
def workspace_reset(app, source, track, no_checkout, element):
"""Reset a workspace to its original state"""
app.initialize((element,))
+ app.pipeline.initialize()
if app.interactive:
if not click.confirm('This will remove all your changes, are you sure?'):
click.echo('Aborting')
@@ -845,9 +856,7 @@ class App():
try:
self.pipeline = Pipeline(self.context, self.project, elements, except_,
- inconsistent=inconsistent,
rewritable=rewritable,
- use_remote_cache=use_remote_cache,
load_ticker=self.load_ticker,
resolve_ticker=self.resolve_ticker,
remote_ticker=self.remote_ticker,
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index fa7e8dc..bd8fbb2 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -53,7 +53,7 @@ class Planner():
# Here we want to traverse the same element more than once when
# it is reachable from multiple places, with the interest of finding
# the deepest occurance of every element
- def plan_element(self, element, depth):
+ def plan_element(self, element, depth, ignore_cache):
if element in self.visiting_elements:
# circular dependency, already being processed
return
@@ -65,22 +65,22 @@ class Planner():
self.visiting_elements.add(element)
for dep in element.dependencies(Scope.RUN, recurse=False):
- self.plan_element(dep, depth)
+ self.plan_element(dep, depth, ignore_cache)
# Dont try to plan builds of elements that are cached already
- if not element._cached() and not element._remotely_cached():
+ if ignore_cache or (not element._cached() and not element._remotely_cached()):
for dep in element.dependencies(Scope.BUILD, recurse=False):
- self.plan_element(dep, depth + 1)
+ self.plan_element(dep, depth + 1, ignore_cache)
self.depth_map[element] = depth
self.visiting_elements.remove(element)
- def plan(self, roots):
+ def plan(self, roots, ignore_cache=False):
for root in roots:
- self.plan_element(root, 0)
+ self.plan_element(root, 0, ignore_cache)
depth_sorted = sorted(self.depth_map.items(), key=itemgetter(1), reverse=True)
- return [item[0] for item in depth_sorted if not item[0]._cached()]
+ return [item[0] for item in depth_sorted if ignore_cache or not item[0]._cached()]
# Pipeline()
@@ -114,13 +114,11 @@ class Planner():
class Pipeline():
def __init__(self, context, project, targets, except_,
- inconsistent=False,
rewritable=False,
- use_remote_cache=False,
- load_ticker=None,
- resolve_ticker=None,
remote_ticker=None,
- cache_ticker=None):
+ cache_ticker=None,
+ load_ticker=None,
+ resolve_ticker=None):
self.context = context
self.project = project
self.session_elements = 0
@@ -128,6 +126,9 @@ class Pipeline():
self.unused_workspaces = []
self._resolved_elements = {}
+ self.remote_ticker = remote_ticker
+ self.cache_ticker = cache_ticker
+
loader = Loader(self.project.element_path, targets + except_,
self.project._options)
meta_elements = loader.load(rewritable, load_ticker)
@@ -154,14 +155,26 @@ class Pipeline():
if resolve_ticker:
resolve_ticker(None)
- # Preflight directly after resolving elements, before ever interrogating
- # caches or anything.
- for plugin in self.dependencies(Scope.ALL, include_sources=True):
- plugin.preflight()
+ def initialize(self, use_remote_cache=False, inconsistent=None):
+ # Preflight directly, before ever interrogating caches or
+ # anything.
+ self.preflight()
self.total_elements = len(list(self.dependencies(Scope.ALL)))
- for element_name, source, workspace in project._list_workspaces():
+ self.initialize_workspaces()
+
+ if use_remote_cache and self.artifacts.can_fetch():
+ self.fetch_remote_refs()
+
+ self.resolve_cache_keys(inconsistent)
+
+ def preflight(self):
+ for plugin in self.dependencies(Scope.ALL, include_sources=True):
+ plugin.preflight()
+
+ def initialize_workspaces(self):
+ for element_name, source, workspace in self.project._list_workspaces():
for target in self.targets:
element = target.search(Scope.ALL, element_name)
@@ -171,21 +184,25 @@ class Pipeline():
self.project._set_workspace(element, source, workspace)
- if use_remote_cache and self.artifacts.can_fetch():
- try:
- if remote_ticker:
- remote_ticker(self.artifacts.url)
- self.artifacts.initialize_remote()
- self.artifacts.fetch_remote_refs()
- except ArtifactError:
- self.message(MessageType.WARN, "Failed to fetch remote refs")
- self.artifacts.set_offline()
+ def fetch_remote_refs(self):
+ try:
+ if self.remote_ticker:
+ self.remote_ticker(self.artifacts.url)
+ self.artifacts.initialize_remote()
+ self.artifacts.fetch_remote_refs()
+ except ArtifactError:
+ self.message(MessageType.WARN, "Failed to fetch remote refs")
+ self.artifacts.set_offline()
+
+ def resolve_cache_keys(self, inconsistent):
+ if inconsistent:
+ inconsistent = self.get_elements_to_track(inconsistent)
for element in self.dependencies(Scope.ALL):
- if cache_ticker:
- cache_ticker(element.name)
+ if self.cache_ticker:
+ self.cache_ticker(element.name)
- if inconsistent:
+ if inconsistent and element in inconsistent:
# Load the pipeline in an explicitly inconsistent state, use
# this for pipelines with tracking queues enabled.
element._force_inconsistent()
@@ -194,8 +211,8 @@ class Pipeline():
# for the first time.
element._cached()
- if cache_ticker:
- cache_ticker(None)
+ if self.cache_ticker:
+ self.cache_ticker(None)
# Generator function to iterate over elements and optionally
# also iterate over sources.
@@ -236,9 +253,11 @@ class Pipeline():
# which are required to build the pipeline target, omitting
# cached elements. The elements are yielded in a depth sorted
# ordering for optimal build plans
- def plan(self):
+ def plan(self, except_=True):
build_plan = Planner().plan(self.targets)
- self.remove_elements(build_plan)
+
+ if except_:
+ build_plan = self.remove_elements(build_plan)
for element in build_plan:
yield element
@@ -304,7 +323,7 @@ class Pipeline():
def track(self, scheduler, dependencies):
dependencies = list(dependencies)
- track = TrackQueue()
+ track = TrackQueue(save=True)
track.enqueue(dependencies)
self.session_elements = len(dependencies)
@@ -376,6 +395,15 @@ class Pipeline():
"Fetched {} elements".format(fetched),
elapsed=elapsed)
+ def get_elements_to_track(self, track_targets):
+ planner = Planner()
+
+ target_elements = [e for e in self.dependencies(Scope.ALL)
+ if e.name in track_targets]
+ track_elements = planner.plan(target_elements, ignore_cache=True)
+
+ return self.remove_elements(track_elements)
+
# build()
#
# Builds (assembles) elements in the pipeline.
@@ -395,15 +423,30 @@ class Pipeline():
detail="\n".join([el + "-" + str(src) for el, src, _
in self.unused_workspaces]))
- if build_all or track_first:
- plan = list(self.dependencies(Scope.ALL))
+ # We set up two plans; one to track elements, the other to
+ # build them once tracking has finished. The first plan
+ # contains elements from track_first, the second contains the
+ # target elements.
+ #
+ # The reason we can't use one plan is that the tracking
+ # elements may consist of entirely different elements.
+ track_plan = []
+ if track_first:
+ track_plan = self.get_elements_to_track(track_first)
+
+ if build_all:
+ plan = self.dependencies(Scope.ALL)
else:
- plan = list(self.plan())
+ plan = self.plan()
- # Assert that we have a consistent pipeline, or that
- # the track option will make it consistent
- if not track_first:
- self.assert_consistent(plan)
+ # We want to start the build queue with any elements that are
+ # not being tracked first
+ track_elements = set(track_plan)
+ plan = [e for e in plan if e not in track_elements]
+
+ # Assert that we have a consistent pipeline now (elements in
+ # track_plan will be made consistent)
+ self.assert_consistent(plan)
fetch = FetchQueue(skip_cached=True)
build = BuildQueue()
@@ -411,7 +454,7 @@ class Pipeline():
pull = None
push = None
queues = []
- if track_first:
+ if track_plan:
track = TrackQueue(save=save)
queues.append(track)
if self.artifacts.can_fetch():
@@ -422,9 +465,14 @@ class Pipeline():
if self.artifacts.can_push():
push = PushQueue()
queues.append(push)
- queues[0].enqueue(plan)
- self.session_elements = len(plan)
+ if track:
+ queues[0].enqueue(track_plan)
+ queues[1].enqueue(plan)
+ else:
+ queues[0].enqueue(plan)
+
+ self.session_elements = len(track_plan) + len(plan)
self.message(MessageType.START, "Starting build")
elapsed, status = scheduler.run(queues)
@@ -771,7 +819,7 @@ class Pipeline():
# use in the result, this function reports a list that is appropriate for
# the selected option.
#
- def deps_elements(self, mode, except_=None):
+ def deps_elements(self, mode):
elements = None
if mode == 'none':