You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:15:26 UTC
[buildstream] 01/01: WIP: Introduce __cache_keys_unstable to handle
workspaces
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jennis/workspace_states
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 62c47baa5634c3aa584eddb1065bdf2ae8759d6f
Author: James Ennis <ja...@codethink.co.uk>
AuthorDate: Thu Jun 27 14:16:31 2019 +0100
WIP: Introduce __cache_keys_unstable to handle workspaces
TODO: Longer comment here
---
src/buildstream/element.py | 52 +++++++++++++++++++++++++++++++++-------------
1 file changed, 38 insertions(+), 14 deletions(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 60aefec..b33b3ab 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -215,6 +215,7 @@ class Element(Plugin):
self.__sources = [] # List of Sources
self.__weak_cache_key = None # Our cached weak cache key
self.__strict_cache_key = None # Our cached cache key for strict builds
+ self.__cache_keys_unstable = None # Whether the current cache keys can be considered as stable
self.__artifacts = context.artifactcache # Artifact cache
self.__sourcecache = context.sourcecache # Source cache
self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state
@@ -1192,17 +1193,20 @@ class Element(Plugin):
# Tracking may still be pending
return
- if self._get_workspace() and self.__assemble_scheduled:
- # If we have an active workspace and are going to build, then
- # discard current cache key values and invoke the buildable callback.
- # The correct keys can only be calculated once the build is complete
- self.__reset_cache_data()
-
- return
-
self.__update_cache_keys()
self.__update_artifact_state()
+ # Workspaces are initially marked with unstable cache keys. Keys will be
+ # marked stable either when we verify that the workspace is already
+ # cached, or when we build/pull the workspaced element.
+ if self.__cache_keys_unstable:
+ if not self._cached_success():
+ self.__reset_cache_data()
+ if not self.__assemble_scheduled:
+ self._schedule_assemble()
+ else:
+ self.__cache_keys_unstable = False
+
# Workspaced sources are considered unstable if a build is pending
# as the build will modify the contents of the workspace.
# Determine as early as possible if a build is pending to discard
@@ -1211,7 +1215,7 @@ class Element(Plugin):
# the cache key.
if (not self.__assemble_scheduled and not self.__assemble_done and
self.__artifact and
- (self._is_required() or self._get_workspace()) and
+ self._is_required() and
not self._cached_success() and
not self._pull_pending()):
self._schedule_assemble()
@@ -1228,7 +1232,7 @@ class Element(Plugin):
if not context.get_strict():
self.__update_cache_key_non_strict()
- if not self.__ready_for_runtime and self.__cache_key is not None:
+ if not self.__ready_for_runtime and self.__cache_key is not None and not self.__cache_keys_unstable:
self.__ready_for_runtime = all(
dep.__ready_for_runtime for dep in self.__runtime_dependencies)
@@ -1584,6 +1588,11 @@ class Element(Plugin):
self.__assemble_scheduled = False
self.__assemble_done = True
+ # If we've just assembled the Element, we are safe to
+ # consider the cache keys as stable
+ if self.__cache_keys_unstable:
+ self.__cache_keys_unstable = False
+
# Artifact may have a cached success now.
if self.__strict_artifact:
self.__strict_artifact.reset_cached()
@@ -1842,6 +1851,11 @@ class Element(Plugin):
def _pull_done(self):
self.__pull_done = True
+ # If we've just successfully pulled the element, we are safe
+ # to consider its keys as stable
+ if self.__cache_keys_unstable and self._cached_success():
+ self.__cache_keys_unstable = False
+
# Artifact may become cached after pulling, so let it query the
# filesystem again to check
self.__strict_artifact.reset_cached()
@@ -2317,11 +2331,12 @@ class Element(Plugin):
# _update_ready_for_runtime_and_cached()
#
- # An Element becomes ready for runtime and cached once the following three criteria
+ # An Element becomes ready for runtime and cached once the following four criteria
# are met:
# 1. The Element has a strong cache key
- # 2. The Element is cached (locally)
- # 3. The runtime dependencies of the Element are ready for runtime and cached.
+ # 2. The Element's keys are considered stable
+ # 3. The Element is cached (locally)
+ # 4. The runtime dependencies of the Element are ready for runtime and cached.
#
# These three criteria serve as potential trigger points as to when an Element may have
# become ready for runtime and cached.
@@ -2332,7 +2347,8 @@ class Element(Plugin):
#
def _update_ready_for_runtime_and_cached(self):
if not self.__ready_for_runtime_and_cached:
- if self.__remaining_runtime_deps_uncached == 0 and self.__cache_key and self._cached_success():
+ if self.__remaining_runtime_deps_uncached == 0 and self._cached_success() and \
+ self.__cache_key and not self.__cache_keys_unstable:
self.__ready_for_runtime_and_cached = True
# Notify reverse dependencies
@@ -3057,6 +3073,14 @@ class Element(Plugin):
def __update_cache_keys(self):
context = self._get_context()
+ # If the Element is workspaced, we should *initially*
+ # consider its keys unstable
+ if self.__cache_keys_unstable is None:
+ if self._get_workspace():
+ self.__cache_keys_unstable = True
+ else:
+ self.__cache_keys_unstable = False
+
if self.__weak_cache_key is None:
# Calculate weak cache key
# Weak cache key includes names of direct build dependencies