You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ju...@apache.org on 2021/03/02 16:02:19 UTC
[buildstream] 04/08: _artifact.py: Make cache query explicit
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/cache-query
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 59e64bd848946a12cf1e2f0950c3b53c21f955d6
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Tue Sep 15 11:07:10 2020 +0200
_artifact.py: Make cache query explicit
Cache query can be fairly expensive as it checks the presence of all
artifact blobs. Make this more explicit with a `query_cache()` method,
instead of implicitly querying the cache on the first call to
`cached()`.
---
src/buildstream/_artifact.py | 29 ++++++++++++++---------------
src/buildstream/_frontend/widget.py | 2 ++
src/buildstream/element.py | 17 ++++++++++-------
3 files changed, 26 insertions(+), 22 deletions(-)
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index d4a716f..b63cff6 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -540,7 +540,7 @@ class Artifact:
return dependency_refs
- # cached():
+ # query_cache():
#
# Check whether the artifact corresponding to the stored cache key is
# available. This also checks whether all required parts of the artifact
@@ -550,11 +550,7 @@ class Artifact:
# Returns:
# (bool): Whether artifact is in local cache
#
- def cached(self):
-
- if self._cached is not None:
- return self._cached
-
+ def query_cache(self):
context = self._context
artifact = self._load_proto()
@@ -587,6 +583,18 @@ class Artifact:
self._cached = True
return True
+ # cached()
+ #
+ # Return whether the artifact is available in the local cache. This must
+ # be called after `query_cache()` or `set_cached()`.
+ #
+ # Returns:
+ # (bool): Whether artifact is in local cache
+ #
+ def cached(self):
+ assert self._cached is not None
+ return self._cached
+
# cached_logs()
#
# Check if the artifact is cached with log files.
@@ -600,15 +608,6 @@ class Artifact:
# If the artifact is cached, its log files are available as well.
return self._element._cached()
- # reset_cached()
- #
- # Allow the Artifact to query the filesystem to determine whether it
- # is cached or not.
- #
- def reset_cached(self):
- self._proto = None
- self._cached = None
-
# set_cached()
#
# Mark the artifact as cached without querying the filesystem.
diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py
index 99c9f42..722fecb 100644
--- a/src/buildstream/_frontend/widget.py
+++ b/src/buildstream/_frontend/widget.py
@@ -357,6 +357,8 @@ class LogLine(Widget):
else:
if element.get_kind() == "junction":
line = p.fmt_subst(line, "state", "junction", fg="magenta")
+ elif not element._can_query_cache():
+ line = p.fmt_subst(line, "state", "unknown", fg="bright_black")
elif element._cached_failure():
line = p.fmt_subst(line, "state", "failed", fg="red")
elif element._cached_success():
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 503361e..087a3fe 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1182,9 +1182,6 @@ class Element(Plugin):
# the artifact cache
#
def _cached(self):
- if not self.__artifact:
- return False
-
return self.__artifact.cached()
# _cached_remotely():
@@ -1300,7 +1297,7 @@ class Element(Plugin):
#
def _can_query_cache(self):
# cache cannot be queried until strict cache key is available
- return self.__strict_cache_key is not None
+ return self.__artifact is not None
# _initialize_state()
#
@@ -1588,7 +1585,9 @@ class Element(Plugin):
def __should_schedule(self):
# We're processing if we're already scheduled, we've
# finished assembling or if we're waiting to pull.
- processing = self.__assemble_scheduled or self.__assemble_done or self._pull_pending()
+ processing = (
+ self.__assemble_scheduled or self.__assemble_done or (self._can_query_cache() and self._pull_pending())
+ )
# We should schedule a build when
return (
@@ -1654,7 +1653,7 @@ class Element(Plugin):
self.__artifact.set_cached()
self.__cached_successfully = True
else:
- self.__artifact.reset_cached()
+ self.__artifact.query_cache()
# When we're building in non-strict mode, we may have
# assembled everything to this point without a strong cache
@@ -1901,7 +1900,7 @@ class Element(Plugin):
# Artifact may become cached after pulling, so let it query the
# filesystem again to check
- self.__artifact.reset_cached()
+ self.__artifact.query_cache()
# We may not have actually pulled an artifact - the pull may
# have failed. We might therefore need to schedule assembly.
@@ -2595,6 +2594,7 @@ class Element(Plugin):
return None
artifact = Artifact(self, self._get_context(), strong_key=workspace.last_build)
+ artifact.query_cache()
if not artifact.cached():
return None
@@ -2868,6 +2868,7 @@ class Element(Plugin):
#
def __initialize_from_artifact(self, artifact: Artifact):
self.__artifact = artifact
+ artifact.query_cache()
self._mimic_artifact()
@classmethod
@@ -3279,12 +3280,14 @@ class Element(Plugin):
strict_key=self.__strict_cache_key,
weak_key=self.__weak_cache_key,
)
+ strict_artifact.query_cache()
if context.get_strict() or strict_artifact.cached():
self.__artifact = strict_artifact
else:
self.__artifact = Artifact(
self, context, strict_key=self.__strict_cache_key, weak_key=self.__weak_cache_key
)
+ self.__artifact.query_cache()
if not context.get_strict() and self.__artifact.cached():
# In non-strict mode, strong cache key becomes available when