You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:55:44 UTC
[buildstream] branch richardmaw/wip/log-show created (now 9328443)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 9328443 WIP: bst-here fixes
This branch includes the following new commits:
new 0fef7bc element.py: Cache the result of checking whether an artifact is cached weakly
new 6c1776d element.py: Add metadata to distinguish between successful and failed builds
new 3c58ac9 Convert call-sites of Element._cached() that assume success
new e894f93 _frontend/widget.py: Render cached failures differently to successes
new 6d55642 Cache failed builds
new 4a57da9 _scheduler/queues/buildqueue.py: Skip rebuilding cached failures
new 0e9b078 tests: Add unit tests for cached behaviours
new 1546ce5 _scheduler/queues: Add failed builds to "done" queue
new 236f995 NEWS: Describe caching of failures
new 67c4874 WIP: Add method for getting build log
new 9bae8ee WIP: Add --logs flag to show
new 9328443 WIP: bst-here fixes
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 08/12: _scheduler/queues: Add failed builds to "done"
queue
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1546ce5ababf25196e824120eec31576423abdec
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Tue Jun 12 17:17:40 2018 +0100
_scheduler/queues: Add failed builds to "done" queue
This allows the scheduler to move jobs from the current queue to the next.
As a result of this change later queues than the build queue
mustn't skip a cached failure, so the logic is specialised to build queues only.
---
buildstream/_scheduler/queues/buildqueue.py | 1 +
buildstream/_scheduler/queues/queue.py | 4 +++-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/buildstream/_scheduler/queues/buildqueue.py b/buildstream/_scheduler/queues/buildqueue.py
index 272f19f..5967fbf 100644
--- a/buildstream/_scheduler/queues/buildqueue.py
+++ b/buildstream/_scheduler/queues/buildqueue.py
@@ -60,6 +60,7 @@ class BuildQueue(Queue):
action_cb=self.process,
complete_cb=self._job_done,
max_retries=self._max_retries)
+ self._done_queue.append(job)
self.failed_elements.append(element)
self._scheduler._job_complete_callback(job, False)
diff --git a/buildstream/_scheduler/queues/queue.py b/buildstream/_scheduler/queues/queue.py
index 6c15834..28da177 100644
--- a/buildstream/_scheduler/queues/queue.py
+++ b/buildstream/_scheduler/queues/queue.py
@@ -296,6 +296,7 @@ class Queue():
# See the Job object for an explanation of the call signature
#
def _job_done(self, job, element, success, result):
+ element._update_state()
# Update values that need to be synchronized in the main task
# before calling any queue implementation
@@ -335,8 +336,9 @@ class Queue():
# No exception occured, handle the success/failure state in the normal way
#
+ self._done_queue.append(job)
+
if success:
- self._done_queue.append(job)
if processed:
self.processed_elements.append(element)
else:
[buildstream] 10/12: WIP: Add method for getting build log
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 67c4874363ec152f28005bdb9519ed3fbe5efe1d
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Thu Jun 14 13:29:37 2018 +0100
WIP: Add method for getting build log
---
buildstream/element.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/buildstream/element.py b/buildstream/element.py
index bde48a4..51fdff3 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1928,6 +1928,11 @@ class Element(Plugin):
def _subst_string(self, value):
return self.__variables.subst(value)
+ def _get_build_log(self):
+ self.__assert_cached()
+ artifact_base, _ = self.__extract()
+ return os.path.join(artifact_base, 'logs', 'build.log')
+
# Returns the element whose sources this element is ultimately derived from.
#
# This is intended for being used to redirect commands that operate on an
[buildstream] 07/12: tests: Add unit tests for cached behaviours
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0e9b0789d451726eef004b1a2da0534bb9120da6
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Tue Jun 5 09:44:21 2018 +0100
tests: Add unit tests for cached behaviours
---
tests/integration/cachedfail.py | 160 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 160 insertions(+)
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
new file mode 100644
index 0000000..f4cabb3
--- /dev/null
+++ b/tests/integration/cachedfail.py
@@ -0,0 +1,160 @@
+import os
+import pytest
+
+from buildstream import _yaml
+from buildstream._exceptions import ErrorDomain
+
+from tests.testutils import cli_integration as cli, create_artifact_share
+from tests.testutils.site import IS_LINUX
+
+
+pytestmark = pytest.mark.integration
+
+
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ "project"
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_build_checkout_cached_fail(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element_path = os.path.join(project, 'elements', 'element.bst')
+ workspace = os.path.join(cli.directory, 'workspace')
+ checkout = os.path.join(cli.directory, 'checkout')
+
+ # Write out our test target
+ element = {
+ 'kind': 'script',
+ 'depends': [
+ {
+ 'filename': 'base.bst',
+ 'type': 'build',
+ },
+ ],
+ 'config': {
+ 'commands': [
+ 'touch %{install-root}/foo',
+ 'false',
+ ],
+ },
+ }
+ _yaml.dump(element, element_path)
+
+ # Try to build it, this should result in a failure that contains the content
+ result = cli.run(project=project, args=['build', 'element.bst'])
+ result.assert_main_error(ErrorDomain.STREAM, None)
+
+ # Assert that it's cached in a failed artifact
+ assert cli.get_element_state(project, 'element.bst') == 'failed'
+
+ # Now check it out
+ result = cli.run(project=project, args=[
+ 'checkout', 'element.bst', checkout
+ ])
+ result.assert_success()
+
+ # Check that the checkout contains the file created before failure
+ filename = os.path.join(checkout, 'foo')
+ assert os.path.exists(filename)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_build_depend_on_cached_fail(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ dep_path = os.path.join(project, 'elements', 'dep.bst')
+ target_path = os.path.join(project, 'elements', 'target.bst')
+ workspace = os.path.join(cli.directory, 'workspace')
+ checkout = os.path.join(cli.directory, 'checkout')
+
+ dep = {
+ 'kind': 'script',
+ 'depends': [
+ {
+ 'filename': 'base.bst',
+ 'type': 'build',
+ },
+ ],
+ 'config': {
+ 'commands': [
+ 'touch %{install-root}/foo',
+ 'false',
+ ],
+ },
+ }
+ _yaml.dump(dep, dep_path)
+ target = {
+ 'kind': 'script',
+ 'depends': [
+ {
+ 'filename': 'base.bst',
+ 'type': 'build',
+ },
+ {
+ 'filename': 'dep.bst',
+ 'type': 'build',
+ },
+ ],
+ 'config': {
+ 'commands': [
+ 'test -e /foo',
+ ],
+ },
+ }
+ _yaml.dump(target, target_path)
+
+ # Try to build it, this should result in caching a failure to build dep
+ result = cli.run(project=project, args=['build', 'dep.bst'])
+ result.assert_main_error(ErrorDomain.STREAM, None)
+
+ # Assert that it's cached in a failed artifact
+ assert cli.get_element_state(project, 'dep.bst') == 'failed'
+
+ # Now we should fail because we've a cached fail of dep
+ result = cli.run(project=project, args=['build', 'target.bst'])
+ result.assert_main_error(ErrorDomain.STREAM, None)
+
+ # Assert that it's not yet built, since one of its dependencies isn't ready.
+ assert cli.get_element_state(project, 'target.bst') == 'waiting'
+
+
+@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("on_error", ("continue",))
+def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element_path = os.path.join(project, 'elements', 'element.bst')
+ workspace = os.path.join(cli.directory, 'workspace')
+ checkout = os.path.join(cli.directory, 'checkout')
+
+ # Write out our test target
+ element = {
+ 'kind': 'script',
+ 'depends': [
+ {
+ 'filename': 'base.bst',
+ 'type': 'build',
+ },
+ ],
+ 'config': {
+ 'commands': [
+ 'false',
+ ],
+ },
+ }
+ _yaml.dump(element, element_path)
+
+ with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as share:
+ cli.configure({
+ 'artifacts': {'url': share.repo, 'push': True},
+ })
+
+ # Build the element, continuing to finish active jobs on error.
+ result = cli.run(project=project, args=['--on-error={}'.format(on_error), 'build', 'element.bst'])
+ result.assert_main_error(ErrorDomain.STREAM, None)
+
+ # This element should have failed
+ assert cli.get_element_state(project, 'element.bst') == 'failed'
+ # This element should have been pushed to the remote
+ assert share.has_artifact('test', 'element.bst', cli.get_element_key(project, 'element.bst'))
[buildstream] 09/12: NEWS: Describe caching of failures
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 236f9958b1f85e9eb78ccbce43ac70cd8b5cebaa
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Tue Jun 5 15:09:47 2018 +0100
NEWS: Describe caching of failures
Closes #76.
---
NEWS | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/NEWS b/NEWS
index dd60fe6..ad730c9 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,13 @@ buildstream 1.3.1
o Add a `--tar` option to `bst checkout` which allows a tarball to be
created from the artifact contents.
+ o Failed builds are included in the cache as well.
+ `bst checkout` will provide anything in `%{install-root}`.
+ A build including cached fails will cause any dependant elements
+ to not be scheduled and fail during artifact assembly,
+ and display the retry prompt during an interactive session.
+
+
=================
buildstream 1.1.4
=================
[buildstream] 03/12: Convert call-sites of Element._cached() that
assume success
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3c58ac98ea63b532e79952cf9c0357dd184c8953
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Jun 4 18:06:40 2018 +0100
Convert call-sites of Element._cached() that assume success
When we later add cached failures it needs to not treat them as successes.
---
buildstream/_frontend/widget.py | 2 +-
buildstream/_pipeline.py | 4 ++--
buildstream/_scheduler/queues/buildqueue.py | 2 +-
buildstream/element.py | 8 ++++----
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index dab8cab..5967b0d 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -368,7 +368,7 @@ class LogLine(Widget):
if consistency == Consistency.INCONSISTENT:
line = p.fmt_subst(line, 'state', "no reference", fg='red')
else:
- if element._cached():
+ if element._cached_success():
line = p.fmt_subst(line, 'state', "cached", fg='magenta')
elif consistency == Consistency.RESOLVED:
line = p.fmt_subst(line, 'state', "fetch needed", fg='red')
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 852abf7..800a331 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -489,7 +489,7 @@ class _Planner():
self.plan_element(dep, depth)
# Dont try to plan builds of elements that are cached already
- if not element._cached():
+ if not element._cached_success():
for dep in element.dependencies(Scope.BUILD, recurse=False):
self.plan_element(dep, depth + 1)
@@ -501,4 +501,4 @@ class _Planner():
self.plan_element(root, 0)
depth_sorted = sorted(self.depth_map.items(), key=itemgetter(1), reverse=True)
- return [item[0] for item in depth_sorted if plan_cached or not item[0]._cached()]
+ return [item[0] for item in depth_sorted if plan_cached or not item[0]._cached_success()]
diff --git a/buildstream/_scheduler/queues/buildqueue.py b/buildstream/_scheduler/queues/buildqueue.py
index 376ef5a..691b9ff 100644
--- a/buildstream/_scheduler/queues/buildqueue.py
+++ b/buildstream/_scheduler/queues/buildqueue.py
@@ -43,7 +43,7 @@ class BuildQueue(Queue):
# Keep it in the queue.
return QueueStatus.WAIT
- if element._cached():
+ if element._cached_success():
return QueueStatus.SKIP
if not element._buildable():
diff --git a/buildstream/element.py b/buildstream/element.py
index d4a70e6..efb646c 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1013,7 +1013,7 @@ class Element(Plugin):
# if the pull job is still pending as the remote cache may have an artifact
# that matches the strict cache key, which is preferred over a locally
# cached artifact with a weak cache key match.
- if not dependency._cached() or not dependency._get_cache_key(strength=_KeyStrength.STRONG):
+ if not dependency._cached_success() or not dependency._get_cache_key(strength=_KeyStrength.STRONG):
return False
if not self.__assemble_scheduled:
@@ -1119,7 +1119,7 @@ class Element(Plugin):
# are sufficient. However, don't update the `cached` attributes
# until the full cache query below.
if (not self.__assemble_scheduled and not self.__assemble_done and
- not self.__is_cached(keystrength=_KeyStrength.WEAK) and
+ not self.__cached_success(keystrength=_KeyStrength.WEAK) and
not self._pull_pending() and self._is_required()):
self._schedule_assemble()
return
@@ -1145,7 +1145,7 @@ class Element(Plugin):
self.__weak_cached = self.__artifacts.contains(self, self.__weak_cache_key)
if (not self.__assemble_scheduled and not self.__assemble_done and
- not self.__cached and not self._pull_pending() and self._is_required()):
+ not self._cached_success() and not self._pull_pending() and self._is_required()):
# Workspaced sources are considered unstable if a build is pending
# as the build will modify the contents of the workspace.
# Determine as early as possible if a build is pending to discard
@@ -1487,7 +1487,7 @@ class Element(Plugin):
def _assemble(self):
# Assert call ordering
- assert not self._cached()
+ assert not self._cached_success()
context = self._get_context()
with self._output_file() as output_file:
[buildstream] 02/12: element.py: Add metadata to distinguish
between successful and failed builds
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6c1776d557efe0d84ac9e04ad099b8aecdb41a70
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Jun 4 18:02:56 2018 +0100
element.py: Add metadata to distinguish between successful and failed builds
This just puts the metadata in place,
we're adding code paths to add failed builds later.
---
buildstream/element.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 84 insertions(+)
diff --git a/buildstream/element.py b/buildstream/element.py
index 9c61ed0..d4a70e6 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -227,6 +227,7 @@ class Element(Plugin):
self.__tainted = None # Whether the artifact is tainted and should not be shared
self.__required = False # Whether the artifact is required in the current session
self.__artifact_size = None # The size of data committed to the artifact cache
+ self.__build_result = None # The result of assembling this Element
# hash tables of loaded artifact metadata, hashed by key
self.__metadata_keys = {} # Strong and weak keys for this key
@@ -953,6 +954,50 @@ class Element(Plugin):
def _cached(self):
return self.__is_cached(keystrength=None)
+ # _get_build_result():
+ #
+ # Returns:
+ # (bool): Whether the artifact of this element present in the artifact cache is of a success
+ # (str): Short description of the result
+ # (str): Detailed description of the result
+ #
+ def _get_build_result(self):
+ return self.__get_build_result(keystrength=None)
+
+ # __set_build_result():
+ #
+ # Sets the assembly result
+ #
+ # Args:
+ # success (bool): Whether the result is a success
+ # description (str): Short description of the result
+ # detail (str): Detailed description of the result
+ #
+ def __set_build_result(self, success, description, detail=None):
+ self.__build_result = (success, description, detail)
+
+ # _cached_success():
+ #
+ # Returns:
+ # (bool): Whether this element is already present in
+ # the artifact cache and the element assembled successfully
+ #
+ def _cached_success(self):
+ return self.__cached_success(keystrength=None)
+
+ # _cached_failure():
+ #
+ # Returns:
+ # (bool): Whether this element is already present in
+ # the artifact cache and the element did not assemble successfully
+ #
+ def _cached_failure(self):
+ if not self._cached():
+ return False
+
+ success, _, _ = self._get_build_result()
+ return not success
+
# _buildable():
#
# Returns:
@@ -1040,6 +1085,7 @@ class Element(Plugin):
self.__strict_cache_key = None
self.__strong_cached = None
self.__weak_cached = None
+ self.__build_result = None
return
if self.__weak_cache_key is None:
@@ -1473,6 +1519,7 @@ class Element(Plugin):
self.__prepare(sandbox)
# Step 4 - Assemble
collect = self.assemble(sandbox)
+ self.__set_build_result(success=True, description="succeeded")
except BstError as e:
# If an error occurred assembling an element in a sandbox,
# then tack on the sandbox directory to the error
@@ -1535,6 +1582,11 @@ class Element(Plugin):
# Store public data
_yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
+ # Store result
+ build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
+ if self.__build_result[2] is not None:
+ build_result_dict["detail"] = self.__build_result[2]
+ _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
# ensure we have cache keys
self._assemble_done()
@@ -2462,6 +2514,38 @@ class Element(Plugin):
metadir = os.path.join(artifact_base, 'meta')
self.__dynamic_public = _yaml.load(os.path.join(metadir, 'public.yaml'))
+ def __load_build_result(self, keystrength):
+ self.__assert_cached(keystrength=keystrength)
+ assert self.__build_result is None
+
+ artifact_base, _ = self.__extract(key=self.__weak_cache_key if keystrength is _KeyStrength.WEAK
+ else self.__strict_cache_key)
+
+ metadir = os.path.join(artifact_base, 'meta')
+ result_path = os.path.join(metadir, 'build-result.yaml')
+ if not os.path.exists(result_path):
+ self.__build_result = (True, "succeeded", None)
+ return
+
+ data = _yaml.load(result_path)
+ self.__build_result = (data["success"], data.get("description"), data.get("detail"))
+
+ def __get_build_result(self, keystrength):
+ if keystrength is None:
+ keystrength = _KeyStrength.STRONG if self._get_context().get_strict() else _KeyStrength.WEAK
+
+ if self.__build_result is None:
+ self.__load_build_result(keystrength)
+
+ return self.__build_result
+
+ def __cached_success(self, keystrength):
+ if not self.__is_cached(keystrength=keystrength):
+ return False
+
+ success, _, _ = self.__get_build_result(keystrength=keystrength)
+ return success
+
def __get_cache_keys_for_commit(self):
keys = []
[buildstream] 01/12: element.py: Cache the result of checking
whether an artifact is cached weakly
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0fef7bcaece0b1e461cb2d0f0d9f47e9a8942cfe
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Jun 4 16:26:26 2018 +0100
element.py: Cache the result of checking whether an artifact is cached weakly
Normally we'd only need it in the case of scheduling a weakly cached build,
but to allow caching of failed builds we need to be able to distinguish
between cached successes and cached failures
for both strong and weak cache keys.
To allow other cache lookup codepaths to look up via the weak key
requires changes through the call stack to consult which key to use,
and cache invalidation of the saved state when it changes.
---
buildstream/element.py | 25 ++++++++++++++++++++-----
1 file changed, 20 insertions(+), 5 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index e9cbdb6..9c61ed0 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -215,6 +215,7 @@ class Element(Plugin):
self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state
self.__cached = None # Whether we have a cached artifact
self.__strong_cached = None # Whether we have a cached artifact
+ self.__weak_cached = None # Whether we have a cached artifact
self.__assemble_scheduled = False # Element is scheduled to be assembled
self.__assemble_done = False # Element is assembled
self.__tracking_scheduled = False # Sources are scheduled to be tracked
@@ -950,7 +951,7 @@ class Element(Plugin):
# the artifact cache
#
def _cached(self):
- return self.__cached
+ return self.__is_cached(keystrength=None)
# _buildable():
#
@@ -1038,6 +1039,7 @@ class Element(Plugin):
self.__weak_cache_key = None
self.__strict_cache_key = None
self.__strong_cached = None
+ self.__weak_cached = None
return
if self.__weak_cache_key is None:
@@ -1060,6 +1062,9 @@ class Element(Plugin):
# Weak cache key could not be calculated yet
return
+ if not self.__weak_cached:
+ self.__weak_cached = self.__artifacts.contains(self, self.__weak_cache_key)
+
if not context.get_strict():
# Full cache query in non-strict mode requires both the weak and
# strict cache keys. However, we need to determine as early as
@@ -1067,9 +1072,9 @@ class Element(Plugin):
# for workspaced elements. For this cache check the weak cache keys
# are sufficient. However, don't update the `cached` attributes
# until the full cache query below.
- cached = self.__artifacts.contains(self, self.__weak_cache_key)
if (not self.__assemble_scheduled and not self.__assemble_done and
- not cached and not self._pull_pending() and self._is_required()):
+ not self.__is_cached(keystrength=_KeyStrength.WEAK) and
+ not self._pull_pending() and self._is_required()):
self._schedule_assemble()
return
@@ -1089,6 +1094,9 @@ class Element(Plugin):
self.__cached = self.__artifacts.contains(self, key_for_cache_lookup)
if not self.__strong_cached:
self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key)
+ if key_for_cache_lookup == self.__weak_cache_key:
+ if not self.__weak_cached:
+ self.__weak_cached = self.__artifacts.contains(self, self.__weak_cache_key)
if (not self.__assemble_scheduled and not self.__assemble_done and
not self.__cached and not self._pull_pending() and self._is_required()):
@@ -1982,12 +1990,19 @@ class Element(Plugin):
if workspace:
workspace.prepared = True
+ def __is_cached(self, keystrength):
+ if keystrength is None:
+ return self.__cached
+
+ return self.__strong_cached if keystrength == _KeyStrength.STRONG else self.__weak_cached
+
# __assert_cached()
#
# Raises an error if the artifact is not cached.
#
- def __assert_cached(self):
- assert self._cached(), "{}: Missing artifact {}".format(self, self._get_brief_display_key())
+ def __assert_cached(self, keystrength=_KeyStrength.STRONG):
+ assert self.__is_cached(keystrength=keystrength), "{}: Missing artifact {}".format(
+ self, self._get_brief_display_key())
# __get_tainted():
#
[buildstream] 12/12: WIP: bst-here fixes
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 932844345d5b0b7c4123e3f50a6ab0b9ce14a571
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Wed Jul 25 16:45:29 2018 +0100
WIP: bst-here fixes
---
contrib/bst-here | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/contrib/bst-here b/contrib/bst-here
index 037788e..c6f17a2 100755
--- a/contrib/bst-here
+++ b/contrib/bst-here
@@ -94,12 +94,12 @@ done
BST_HERE_PS1="\[\033[01;34m\]\w\[\033[00m\]> "
if [ "$#" -eq 0 ]; then
- command="/bin/bash -i"
+ set -- /bin/bash -i
else
- command="/usr/local/bin/bst $@"
+ set -- /bin/sh -c 'exec bst "$@"' - "$@"
fi
-if "$update" == true
+if [ "$update" = true ]
then
docker pull "$bst_here_image"
fi
@@ -123,4 +123,4 @@ exec docker run --rm -i${is_tty:+ -t} \
$extra_volumes_opt \
--workdir /src \
"$bst_here_image" \
- $command
+ "$@"
[buildstream] 11/12: WIP: Add --logs flag to show
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9bae8ee8c896fb20441b8e964505b8fa645ddd27
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Thu Jun 14 13:52:01 2018 +0100
WIP: Add --logs flag to show
This is also possible by adding %{build-log} to the format string,
but --logs to just append '\n%{build-log}' has been added for convenience.
---
buildstream/_frontend/cli.py | 8 +++++++-
buildstream/_frontend/widget.py | 11 +++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index bd2ce8a..3078574 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -491,10 +491,12 @@ def push(app, elements, deps, remote):
@click.option('--format', '-f', 'format_', metavar='FORMAT', default=None,
type=click.STRING,
help='Format string for each element')
+@click.option('--logs', 'logs', default=False,
+ is_flag=True, help='Display logs in summary')
@click.argument('elements', nargs=-1,
type=click.Path(readable=False))
@click.pass_obj
-def show(app, elements, deps, except_, order, format_):
+def show(app, elements, deps, except_, order, format_, logs):
"""Show elements in the pipeline
By default this will show all of the dependencies of the
@@ -526,6 +528,7 @@ def show(app, elements, deps, except_, order, format_):
%{public} Public domain data
%{workspaced} If the element is workspaced
%{workspace-dirs} A list of workspace directories
+ %{build-log} Content of build log
The value of the %{symbol} without the leading '%' character is understood
as a pythonic formatting string, so python formatting features apply,
@@ -552,6 +555,9 @@ def show(app, elements, deps, except_, order, format_):
if not format_:
format_ = app.context.log_element_format
+ if logs:
+ format_ += "\n%{build-log}"
+
report = app.logger.show_pipeline(dependencies, format_)
click.echo(report, color=app.colors)
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index c43dce6..d755a02 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -423,6 +423,17 @@ class LogLine(Widget):
line = p.fmt_subst(
line, 'workspace-dirs', '')
+ # Logs
+ if "%{build-log" in format_:
+ if element._cached():
+ log_path = element._get_build_log()
+ content = open(log_path, 'r').read()
+ line = p.fmt_subst(
+ line, 'build-log', content)
+ else:
+ line = p.fmt_subst(
+ line, 'build-log', '')
+
report += line + '\n'
return report.rstrip('\n')
[buildstream] 05/12: Cache failed builds
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6d55642251a66bb4290a8d96ada91699fd739d95
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Jun 4 18:12:56 2018 +0100
Cache failed builds
This creates an artifact when element assembly fails too,
and if it's the right kind of exception uses the now-included install directory
similarly to if it had returned successfully.
If there's a failure during install the artifact contains any installed files,
but may contain nothing at all.
---
buildstream/buildelement.py | 3 +-
buildstream/element.py | 160 +++++++++++++++++++++++--------------------
buildstream/scriptelement.py | 3 +-
3 files changed, 89 insertions(+), 77 deletions(-)
diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py
index ec05acb..b91cae2 100644
--- a/buildstream/buildelement.py
+++ b/buildstream/buildelement.py
@@ -240,4 +240,5 @@ class BuildElement(Element):
exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
SandboxFlags.ROOT_READ_ONLY)
if exitcode != 0:
- raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode))
+ raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
+ collect=self.get_variable('install-root'))
diff --git a/buildstream/element.py b/buildstream/element.py
index efb646c..bde48a4 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -140,10 +140,13 @@ class ElementError(BstError):
message (str): The error message to report to the user
detail (str): A possibly multiline, more detailed error message
reason (str): An optional machine readable reason string, used for test cases
+ collect (str): An optional directory containing partial install contents
"""
- def __init__(self, message, *, detail=None, reason=None):
+ def __init__(self, message, *, detail=None, reason=None, collect=None):
super().__init__(message, detail=detail, domain=ErrorDomain.ELEMENT, reason=reason)
+ self.collect = collect
+
class Element(Plugin):
"""Element()
@@ -1510,6 +1513,7 @@ class Element(Plugin):
self.__dynamic_public = _yaml.node_copy(self.__public)
# Call the abstract plugin methods
+ collect = None
try:
# Step 1 - Configure
self.configure_sandbox(sandbox)
@@ -1543,81 +1547,87 @@ class Element(Plugin):
self.warn("Failed to preserve workspace state for failed build sysroot: {}"
.format(e))
- raise
+ if isinstance(e, ElementError):
+ collect = e.collect # pylint: disable=no-member
- collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep))
- if not os.path.exists(collectdir):
- raise ElementError(
- "Directory '{}' was not found inside the sandbox, "
- "unable to collect artifact contents"
- .format(collect))
-
- # At this point, we expect an exception was raised leading to
- # an error message, or we have good output to collect.
-
- # Create artifact directory structure
- assembledir = os.path.join(rootdir, 'artifact')
- filesdir = os.path.join(assembledir, 'files')
- logsdir = os.path.join(assembledir, 'logs')
- metadir = os.path.join(assembledir, 'meta')
- buildtreedir = os.path.join(assembledir, 'buildtree')
- os.mkdir(assembledir)
- os.mkdir(filesdir)
- os.mkdir(logsdir)
- os.mkdir(metadir)
- os.mkdir(buildtreedir)
-
- # Hard link files from collect dir to files directory
- utils.link_files(collectdir, filesdir)
-
- sandbox_build_dir = os.path.join(sandbox_root, self.get_variable('build-root').lstrip(os.sep))
- # Hard link files from build-root dir to buildtreedir directory
- if os.path.isdir(sandbox_build_dir):
- utils.link_files(sandbox_build_dir, buildtreedir)
-
- # Copy build log
- log_filename = context.get_log_filename()
- if log_filename:
- shutil.copyfile(log_filename, os.path.join(logsdir, 'build.log'))
-
- # Store public data
- _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
- # Store result
- build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
- if self.__build_result[2] is not None:
- build_result_dict["detail"] = self.__build_result[2]
- _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
-
- # ensure we have cache keys
- self._assemble_done()
-
- # Store keys.yaml
- _yaml.dump(_yaml.node_sanitize({
- 'strong': self._get_cache_key(),
- 'weak': self._get_cache_key(_KeyStrength.WEAK),
- }), os.path.join(metadir, 'keys.yaml'))
-
- # Store dependencies.yaml
- _yaml.dump(_yaml.node_sanitize({
- e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
- }), os.path.join(metadir, 'dependencies.yaml'))
-
- # Store workspaced.yaml
- _yaml.dump(_yaml.node_sanitize({
- 'workspaced': True if self._get_workspace() else False
- }), os.path.join(metadir, 'workspaced.yaml'))
-
- # Store workspaced-dependencies.yaml
- _yaml.dump(_yaml.node_sanitize({
- 'workspaced-dependencies': [
- e.name for e in self.dependencies(Scope.BUILD)
- if e._get_workspace()
- ]
- }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
-
- with self.timed_activity("Caching artifact"):
- self.__artifact_size = utils._get_dir_size(assembledir)
- self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
+ self.__set_build_result(success=False, description=str(e), detail=e.detail)
+ raise
+ finally:
+ if collect is not None:
+ collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep))
+
+ # Create artifact directory structure
+ assembledir = os.path.join(rootdir, 'artifact')
+ filesdir = os.path.join(assembledir, 'files')
+ logsdir = os.path.join(assembledir, 'logs')
+ metadir = os.path.join(assembledir, 'meta')
+ buildtreedir = os.path.join(assembledir, 'buildtree')
+ os.mkdir(assembledir)
+ if collect is not None and os.path.exists(collectdir):
+ os.mkdir(filesdir)
+ os.mkdir(logsdir)
+ os.mkdir(metadir)
+ os.mkdir(buildtreedir)
+
+ # Hard link files from collect dir to files directory
+ if collect is not None and os.path.exists(collectdir):
+ utils.link_files(collectdir, filesdir)
+
+ sandbox_build_dir = os.path.join(sandbox_root, self.get_variable('build-root').lstrip(os.sep))
+ # Hard link files from build-root dir to buildtreedir directory
+ if os.path.isdir(sandbox_build_dir):
+ utils.link_files(sandbox_build_dir, buildtreedir)
+
+ # Copy build log
+ log_filename = context.get_log_filename()
+ if log_filename:
+ shutil.copyfile(log_filename, os.path.join(logsdir, 'build.log'))
+
+ # Store public data
+ _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
+
+ # Store result
+ build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
+ if self.__build_result[2] is not None:
+ build_result_dict["detail"] = self.__build_result[2]
+ _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
+
+ # ensure we have cache keys
+ self._assemble_done()
+
+ # Store keys.yaml
+ _yaml.dump(_yaml.node_sanitize({
+ 'strong': self._get_cache_key(),
+ 'weak': self._get_cache_key(_KeyStrength.WEAK),
+ }), os.path.join(metadir, 'keys.yaml'))
+
+ # Store dependencies.yaml
+ _yaml.dump(_yaml.node_sanitize({
+ e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
+ }), os.path.join(metadir, 'dependencies.yaml'))
+
+ # Store workspaced.yaml
+ _yaml.dump(_yaml.node_sanitize({
+ 'workspaced': True if self._get_workspace() else False
+ }), os.path.join(metadir, 'workspaced.yaml'))
+
+ # Store workspaced-dependencies.yaml
+ _yaml.dump(_yaml.node_sanitize({
+ 'workspaced-dependencies': [
+ e.name for e in self.dependencies(Scope.BUILD)
+ if e._get_workspace()
+ ]
+ }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
+
+ with self.timed_activity("Caching artifact"):
+ self.__artifact_size = utils._get_dir_size(assembledir)
+ self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
+
+ if collect is not None and not os.path.exists(collectdir):
+ raise ElementError(
+ "Directory '{}' was not found inside the sandbox, "
+ "unable to collect artifact contents"
+ .format(collect))
# Finally cleanup the build dir
cleanup_rootdir()
diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py
index 46afda8..145dc26 100644
--- a/buildstream/scriptelement.py
+++ b/buildstream/scriptelement.py
@@ -277,7 +277,8 @@ class ScriptElement(Element):
exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
SandboxFlags.ROOT_READ_ONLY if self.__root_read_only else 0)
if exitcode != 0:
- raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode))
+ raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
+ collect=self.__install_root)
# Return where the result can be collected from
return self.__install_root
[buildstream] 06/12: _scheduler/queues/buildqueue.py: Skip
rebuilding cached failures
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4a57da948e537483111777bf81ad6a53ad379d30
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Jun 4 18:17:34 2018 +0100
_scheduler/queues/buildqueue.py: Skip rebuilding cached failures
This flags up a failure and if run in an interactive prompt
permits the user to attempt a rebuild.
---
buildstream/_scheduler/queues/buildqueue.py | 35 +++++++++++++++++++++++++++++
1 file changed, 35 insertions(+)
diff --git a/buildstream/_scheduler/queues/buildqueue.py b/buildstream/_scheduler/queues/buildqueue.py
index 691b9ff..272f19f 100644
--- a/buildstream/_scheduler/queues/buildqueue.py
+++ b/buildstream/_scheduler/queues/buildqueue.py
@@ -18,8 +18,12 @@
# Tristan Van Berkom <tr...@codethink.co.uk>
# Jürg Billeter <ju...@codethink.co.uk>
+from datetime import timedelta
+
from . import Queue, QueueStatus
+from ..jobs import ElementJob
from ..resources import ResourceType
+from ..._message import MessageType
# A queue which assembles elements
@@ -30,6 +34,37 @@ class BuildQueue(Queue):
complete_name = "Built"
resources = [ResourceType.PROCESS]
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._tried = set()
+
+ def enqueue(self, elts):
+ to_queue = []
+
+ for element in elts:
+ if not element._cached_failure() or element in self._tried:
+ to_queue.append(element)
+ continue
+
+ # Bypass queue processing entirely the first time it's tried.
+ self._tried.add(element)
+ _, description, detail = element._get_build_result()
+ logfile = element._get_build_log()
+ self._message(element, MessageType.FAIL, description,
+ detail=detail, action_name=self.action_name,
+ elapsed=timedelta(seconds=0),
+ logfile=logfile)
+ job = ElementJob(self._scheduler, self.action_name,
+ logfile, element=element, queue=self,
+ resources=self.resources,
+ action_cb=self.process,
+ complete_cb=self._job_done,
+ max_retries=self._max_retries)
+ self.failed_elements.append(element)
+ self._scheduler._job_complete_callback(job, False)
+
+ return super().enqueue(to_queue)
+
def process(self, element):
element._assemble()
return element._get_unique_id()
[buildstream] 04/12: _frontend/widget.py: Render cached failures
differently to successes
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip/log-show
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e894f93628a2892de2b68e121879bf86360ee6bd
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Mon Jun 4 18:11:52 2018 +0100
_frontend/widget.py: Render cached failures differently to successes
---
buildstream/_frontend/widget.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index 5967b0d..c43dce6 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -368,7 +368,9 @@ class LogLine(Widget):
if consistency == Consistency.INCONSISTENT:
line = p.fmt_subst(line, 'state', "no reference", fg='red')
else:
- if element._cached_success():
+ if element._cached_failure():
+ line = p.fmt_subst(line, 'state', "failed", fg='red')
+ elif element._cached_success():
line = p.fmt_subst(line, 'state', "cached", fg='magenta')
elif consistency == Consistency.RESOLVED:
line = p.fmt_subst(line, 'state', "fetch needed", fg='red')