You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:15:25 UTC
[buildstream] branch jennis/workspace_states created (now 62c47ba)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch jennis/workspace_states
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 62c47ba WIP: Introduce __cache_keys_unstable to handle workspaces
This branch includes the following new commits:
new 62c47ba WIP: Introduce __cache_keys_unstable to handle workspaces
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: WIP: Introduce __cache_keys_unstable to handle
workspaces
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jennis/workspace_states
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 62c47baa5634c3aa584eddb1065bdf2ae8759d6f
Author: James Ennis <ja...@codethink.co.uk>
AuthorDate: Thu Jun 27 14:16:31 2019 +0100
WIP: Introduce __cache_keys_unstable to handle workspaces
TODO: Longer comment here
---
src/buildstream/element.py | 52 +++++++++++++++++++++++++++++++++-------------
1 file changed, 38 insertions(+), 14 deletions(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 60aefec..b33b3ab 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -215,6 +215,7 @@ class Element(Plugin):
self.__sources = [] # List of Sources
self.__weak_cache_key = None # Our cached weak cache key
self.__strict_cache_key = None # Our cached cache key for strict builds
+ self.__cache_keys_unstable = None # Whether the current cache keys can be considered as stable
self.__artifacts = context.artifactcache # Artifact cache
self.__sourcecache = context.sourcecache # Source cache
self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state
@@ -1192,17 +1193,20 @@ class Element(Plugin):
# Tracking may still be pending
return
- if self._get_workspace() and self.__assemble_scheduled:
- # If we have an active workspace and are going to build, then
- # discard current cache key values and invoke the buildable callback.
- # The correct keys can only be calculated once the build is complete
- self.__reset_cache_data()
-
- return
-
self.__update_cache_keys()
self.__update_artifact_state()
+ # Workspaces are initially marked with unstable cache keys. Keys will be
+ # marked stable either when we verify that the workspace is already
+ # cached, or when we build/pull the workspaced element.
+ if self.__cache_keys_unstable:
+ if not self._cached_success():
+ self.__reset_cache_data()
+ if not self.__assemble_scheduled:
+ self._schedule_assemble()
+ else:
+ self.__cache_keys_unstable = False
+
# Workspaced sources are considered unstable if a build is pending
# as the build will modify the contents of the workspace.
# Determine as early as possible if a build is pending to discard
@@ -1211,7 +1215,7 @@ class Element(Plugin):
# the cache key.
if (not self.__assemble_scheduled and not self.__assemble_done and
self.__artifact and
- (self._is_required() or self._get_workspace()) and
+ self._is_required() and
not self._cached_success() and
not self._pull_pending()):
self._schedule_assemble()
@@ -1228,7 +1232,7 @@ class Element(Plugin):
if not context.get_strict():
self.__update_cache_key_non_strict()
- if not self.__ready_for_runtime and self.__cache_key is not None:
+ if not self.__ready_for_runtime and self.__cache_key is not None and not self.__cache_keys_unstable:
self.__ready_for_runtime = all(
dep.__ready_for_runtime for dep in self.__runtime_dependencies)
@@ -1584,6 +1588,11 @@ class Element(Plugin):
self.__assemble_scheduled = False
self.__assemble_done = True
+ # If we've just assembled the Element, we are safe to
+ # consider the cache keys as stable
+ if self.__cache_keys_unstable:
+ self.__cache_keys_unstable = False
+
# Artifact may have a cached success now.
if self.__strict_artifact:
self.__strict_artifact.reset_cached()
@@ -1842,6 +1851,11 @@ class Element(Plugin):
def _pull_done(self):
self.__pull_done = True
+ # If we've just successfully pulled the element, we are safe
+ # to consider its keys as stable
+ if self.__cache_keys_unstable and self._cached_success():
+ self.__cache_keys_unstable = False
+
# Artifact may become cached after pulling, so let it query the
# filesystem again to check
self.__strict_artifact.reset_cached()
@@ -2317,11 +2331,12 @@ class Element(Plugin):
# _update_ready_for_runtime_and_cached()
#
- # An Element becomes ready for runtime and cached once the following three criteria
+ # An Element becomes ready for runtime and cached once the following four criteria
# are met:
# 1. The Element has a strong cache key
- # 2. The Element is cached (locally)
- # 3. The runtime dependencies of the Element are ready for runtime and cached.
+ # 2. The Element's keys are considered stable
+ # 3. The Element is cached (locally)
+ # 4. The runtime dependencies of the Element are ready for runtime and cached.
#
# These three criteria serve as potential trigger points as to when an Element may have
# become ready for runtime and cached.
@@ -2332,7 +2347,8 @@ class Element(Plugin):
#
def _update_ready_for_runtime_and_cached(self):
if not self.__ready_for_runtime_and_cached:
- if self.__remaining_runtime_deps_uncached == 0 and self.__cache_key and self._cached_success():
+ if self.__remaining_runtime_deps_uncached == 0 and self._cached_success() and \
+ self.__cache_key and not self.__cache_keys_unstable:
self.__ready_for_runtime_and_cached = True
# Notify reverse dependencies
@@ -3057,6 +3073,14 @@ class Element(Plugin):
def __update_cache_keys(self):
context = self._get_context()
+ # If the Element is workspaced, we should *initially*
+ # consider its keys unstable
+ if self.__cache_keys_unstable is None:
+ if self._get_workspace():
+ self.__cache_keys_unstable = True
+ else:
+ self.__cache_keys_unstable = False
+
if self.__weak_cache_key is None:
# Calculate weak cache key
# Weak cache key includes names of direct build dependencies