[buildstream] 06/12: _elementsources.py: Make cache query explicit

[tv...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch juerg/cache-query-job-benchmark
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 75df8f827462a1e966be40dd5575941784baaa85
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu Sep 17 11:04:15 2020 +0200

    _elementsources.py: Make cache query explicit
    
    Cache query can be fairly expensive as it checks the presence of all
    blobs. Make this more explicit with a `query_cache()` method, instead of
    implicitly querying the cache on the first call to `cached()`.
---
 src/buildstream/_elementsources.py  | 31 ++++++++++++++++++++++++++-----
 src/buildstream/_frontend/widget.py |  2 ++
 src/buildstream/element.py          | 10 ++++++++++
 3 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/src/buildstream/_elementsources.py b/src/buildstream/_elementsources.py
index a15d20e..9b4afe4 100644
--- a/src/buildstream/_elementsources.py
+++ b/src/buildstream/_elementsources.py
@@ -293,7 +293,7 @@ class ElementSources:
         length = min(len(key), context.log_key_length)
         return key[:length]
 
-    # cached():
+    # query_cache():
     #
     # Check if the element sources are cached in CAS, generating the source
     # cache keys if needed.
@@ -301,10 +301,7 @@ class ElementSources:
     # Returns:
     #    (bool): True if the element sources are cached
     #
-    def cached(self):
-        if self._cached is not None:
-            return self._cached
-
+    def query_cache(self):
         cas = self._context.get_cascache()
         elementsourcescache = self._elementsourcescache
 
@@ -321,6 +318,28 @@ class ElementSources:
         self._cached = True
         return True
 
+    # can_query_cache():
+    #
+    # Returns whether the cache status is available.
+    #
+    # Returns:
+    #    (bool): True if cache status is available
+    #
+    def can_query_cache(self):
+        return self._cached is not None
+
+    # cached()
+    #
+    # Return whether the element sources are cached in CAS. This must be
+    # called only when all sources are resolved.
+    #
+    # Returns:
+    #    (bool): True if the element sources are cached
+    #
+    def cached(self):
+        assert self._cached is not None
+        return self._cached
+
     # is_resolved():
     #
     # Get whether all sources of the element are resolved
@@ -368,6 +387,8 @@ class ElementSources:
         unique_key = self.get_unique_key()
         self._cache_key = _cachekey.generate_key(unique_key)
 
+        self.query_cache()
+
     # preflight():
     #
     # A internal wrapper for calling the abstract preflight() method on
diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py
index d289ef2..ad6c813 100644
--- a/src/buildstream/_frontend/widget.py
+++ b/src/buildstream/_frontend/widget.py
@@ -363,6 +363,8 @@ class LogLine(Widget):
                         line = p.fmt_subst(line, "state", "failed", fg="red")
                     elif element._cached_success():
                         line = p.fmt_subst(line, "state", "cached", fg="magenta")
+                    elif not element._can_query_source_cache():
+                        line = p.fmt_subst(line, "state", "waiting", fg="blue")
                     elif element._fetch_needed():
                         line = p.fmt_subst(line, "state", "fetch needed", fg="red")
                     elif element._buildable():
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 0f2a01c..c4f0479 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1296,6 +1296,16 @@ class Element(Plugin):
         # cache cannot be queried until strict cache key is available
         return self.__artifact is not None
 
+    # _can_query_source_cache():
+    #
+    # Returns whether the source cache status is available.
+    #
+    # Returns:
+    #    (bool): True if source cache can be queried
+    #
+    def _can_query_source_cache(self):
+        return self.__sources.can_query_cache()
+
     # _initialize_state()
     #
     # Compute up the elment's initial state. Element state contains