You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/02/22 07:59:32 UTC
[buildstream] branch tristan/include-weak-key-in-cache-key created (now 4775e9b)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch tristan/include-weak-key-in-cache-key
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 4775e9b element.py: Include weak cache key in cache key calculation.
This branch includes the following new commits:
new 4775e9b element.py: Include weak cache key in cache key calculation.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: element.py: Include weak cache key in cache key calculation.
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/include-weak-key-in-cache-key
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4775e9be332a6b11a294c26446fd202100c01971
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Feb 22 16:55:14 2022 +0900
element.py: Include weak cache key in cache key calculation.
This should eliminate a class of issues related to strict/non-strict
discrepency.
This also updates the cache key test.
Fixes #1270
---
src/buildstream/element.py | 55 +++++++++++++----------
tests/cachekey/project/elements/build1.expected | 2 +-
tests/cachekey/project/elements/build2.expected | 2 +-
tests/cachekey/project/elements/build3.expected | 2 +-
tests/cachekey/project/elements/compose1.expected | 2 +-
tests/cachekey/project/elements/compose2.expected | 2 +-
tests/cachekey/project/elements/compose3.expected | 2 +-
tests/cachekey/project/elements/compose4.expected | 2 +-
tests/cachekey/project/elements/compose5.expected | 2 +-
tests/cachekey/project/elements/import1.expected | 2 +-
tests/cachekey/project/elements/import2.expected | 2 +-
tests/cachekey/project/elements/import3.expected | 2 +-
tests/cachekey/project/elements/script1.expected | 2 +-
tests/cachekey/project/sources/bzr1.expected | 2 +-
tests/cachekey/project/sources/git1.expected | 2 +-
tests/cachekey/project/sources/git2.expected | 2 +-
tests/cachekey/project/sources/git3.expected | 2 +-
tests/cachekey/project/sources/local1.expected | 2 +-
tests/cachekey/project/sources/local2.expected | 2 +-
tests/cachekey/project/sources/patch1.expected | 2 +-
tests/cachekey/project/sources/patch2.expected | 2 +-
tests/cachekey/project/sources/patch3.expected | 2 +-
tests/cachekey/project/sources/pip1.expected | 2 +-
tests/cachekey/project/sources/remote1.expected | 2 +-
tests/cachekey/project/sources/remote2.expected | 2 +-
tests/cachekey/project/sources/tar1.expected | 2 +-
tests/cachekey/project/sources/tar2.expected | 2 +-
tests/cachekey/project/sources/zip1.expected | 2 +-
tests/cachekey/project/sources/zip2.expected | 2 +-
tests/cachekey/project/target.expected | 2 +-
30 files changed, 61 insertions(+), 52 deletions(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index ee7e76c..e891349 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -2200,13 +2200,15 @@ class Element(Plugin):
# Args:
# dependencies (List[List[str]]): list of dependencies with project name,
# element name and optional cache key
+ # weak_cache_key (Optional[str]): the weak cache key, required for calculating the
+ # strict and strong cache keys
#
# Returns:
# (str): A hex digest cache key for this Element, or None
#
# None is returned if information for the cache key is missing.
#
- def _calculate_cache_key(self, dependencies):
+ def _calculate_cache_key(self, dependencies, weak_cache_key=None):
# No cache keys for dependencies which have no cache keys
if any(not all(dep) for dep in dependencies):
return None
@@ -2235,6 +2237,8 @@ class Element(Plugin):
cache_key_dict = self.__cache_key_dict.copy()
cache_key_dict["dependencies"] = dependencies
+ if weak_cache_key is not None:
+ cache_key_dict["weak-cache-key"] = weak_cache_key
return _cachekey.generate_key(cache_key_dict)
@@ -3219,37 +3223,40 @@ class Element(Plugin):
# Tracking may still be pending
return
+ # Calculate weak cache key first, as it is required for generating the other keys.
+ #
+ # This code can be run multiple times until the strict key can be calculated,
+ # so let's ensure we only ever calculate the weak key once, even though we need
+ # to resolve it before we can resolve the strict key.
+ if self.__weak_cache_key is None:
+ # Weak cache key includes names of direct build dependencies
+ # so as to only trigger rebuilds when the shape of the
+ # dependencies change.
+ #
+ # Some conditions cause dependencies to be strict, such
+ # that this element will be rebuilt anyway if the dependency
+ # changes even in non strict mode, for these cases we just
+ # encode the dependency's weak cache key instead of it's name.
+ #
+ dependencies = [
+ [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)]
+ if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
+ else [e.project_name, e.name]
+ for e in self._dependencies(_Scope.BUILD)
+ ]
+ self.__weak_cache_key = self._calculate_cache_key(dependencies)
+
context = self._get_context()
# Calculate the strict cache key
dependencies = [[e.project_name, e.name, e.__strict_cache_key] for e in self._dependencies(_Scope.BUILD)]
- self.__strict_cache_key = self._calculate_cache_key(dependencies)
+ self.__strict_cache_key = self._calculate_cache_key(dependencies, self.__weak_cache_key)
if self.__strict_cache_key is None:
# Cache keys cannot be calculated yet as a build dependency doesn't
# have a cache key yet.
return
- # Calculate weak cache key
- #
- # Weak cache key includes names of direct build dependencies
- # so as to only trigger rebuilds when the shape of the
- # dependencies change.
- #
- # Some conditions cause dependencies to be strict, such
- # that this element will be rebuilt anyway if the dependency
- # changes even in non strict mode, for these cases we just
- # encode the dependency's weak cache key instead of it's name.
- #
- dependencies = [
- [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)]
- if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
- else [e.project_name, e.name]
- for e in self._dependencies(_Scope.BUILD)
- ]
-
- self.__weak_cache_key = self._calculate_cache_key(dependencies)
-
# As the strict cache key has already been calculated, it should always
# be possible to calculate the weak cache key as well.
assert self.__weak_cache_key is not None
@@ -3287,9 +3294,11 @@ class Element(Plugin):
strong_key, _, _ = self.__artifact.get_metadata_keys()
self.__cache_key = strong_key
elif self.__assemble_scheduled or self.__assemble_done:
+ assert self.__weak_cache_key is not None
+
# Artifact will or has been built, not downloaded
dependencies = [[e.project_name, e.name, e._get_cache_key()] for e in self._dependencies(_Scope.BUILD)]
- self.__cache_key = self._calculate_cache_key(dependencies)
+ self.__cache_key = self._calculate_cache_key(dependencies, self.__weak_cache_key)
if self.__cache_key is None:
# Strong cache key could not be calculated yet
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index f9e425c..98311d4 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-df973166242fc7613903e09b39f8c44990868af0554421ee7c1beaf9fc268030
\ No newline at end of file
+7f00e0e1d2f7f154895c0392816750984ddcbcde290a87df475a3ff1ef151308
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index 9f0a4d2..f07a044 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-0d3d9cb2c3d8569806f4822a2bd52150a675d4621883b35e02f266cfc10bc457
\ No newline at end of file
+11578c1823d8fad0143bb1afd195a6dbc5b36a11834ab04d1babe7d5750b5341
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build3.expected b/tests/cachekey/project/elements/build3.expected
index fd339a4..2edc024 100644
--- a/tests/cachekey/project/elements/build3.expected
+++ b/tests/cachekey/project/elements/build3.expected
@@ -1 +1 @@
-8fbfdfed830260389b941583dcde6533202a3e970063aa6569dfd1b4dd303356
\ No newline at end of file
+bf6067aaba914c449080a8e477d8b206dd3a756085874be4ec0d902850d3f0a1
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index fd88e67..8f769e5 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-f3eb266781b7cea1230d5def8f49d21312ae98e2156b77ff429690ebefcf6d5a
\ No newline at end of file
+79741eae7ac50457a67f06d2913f549080739dada5033b3015fb68b494536c5a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index d979a89..1d1db39 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-827b5de4a4cc7cb2a0738d792287af56ca95e29da75aea5c7674cb6fa9d06b70
\ No newline at end of file
+329f4ff2c3855f541ff2f85d0ba5ad4a87c73610914f84483c8462b095a8007e
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 291e94c..d56d567 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-82e9499af43b8ce933e61b2b93f342930616ac06f2288ea8b00c9245935beb67
\ No newline at end of file
+db8429d5f2a87335cfb3ca005b13a12267b3ba4ef369c72ee2d6dd237c4cf12a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 388fe77..f7c85c1 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-77ff7b5a2286bce2dbe2b6bb7e98908d709c5b00450af5c8e9221eccd481eae2
\ No newline at end of file
+fff3c6f147a40ba9b47a4f6b39fc307934db891659d3d06258ab4b2458a5650f
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index d7625b0..57ddda2 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-6239ae1b6b33c6bdb3b25198c2d4a43d0c77dd7d95f540a42e139188a074dcfd
\ No newline at end of file
+7040e4b5a4b82f2c4c511e62a125239dbb07ac171673ba167ab9e39d4e491763
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index 5f013e6..a3133cc 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-2e13c26448868c7c30cf7e4549627d757cadab2901d578f40acdd17aaa535816
\ No newline at end of file
+3b62fa381e9183353cac12f8e6ced23ea42931ed4f89ffe24625b6ec9f236ab5
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 0ed5739..4d32ff0 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-4b2a646a93b955392449d8e9c4dcb121e52e1698a11859d90df12d08da8a4564
\ No newline at end of file
+abeff9a432127674cff36acb36f45cd2665eebf75b589693db6676885e37c59c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index f071fe2..0fc1c1f 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-71b8113e587bb9e6549bd95b71dac7b5d2c7d7d2de59f3f27c247fec860a55b5
\ No newline at end of file
+f2c5ae1c73a0d8abfbffb36b09bd74929e9b680bdb548f5faa1213a4c0423fe8
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index f7d213b..2eeb3e0 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-4b5a4267ba32fca936acf7212213e03c5a6f0448f66444f54009243b5425ca3c
\ No newline at end of file
+eb81c0dfd2432b1ebf0bb052d6fa3a529438614c00210cfdc276bbcc2f3045d5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 3a344db..dcad774 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-a19ed9f67c1106b3af0e43dd2e4485b7cc0ef40a26c39f76ae102a2791dac0b0
\ No newline at end of file
+a8ca1b3d852de71c0d97fbdddfe86f815a8edc62acf5aebbdcca1972a1627b12
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 66594a0..7cb0871 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-d554f77e28b68c4085603e67dc3db69bdedb7284efceee60f7bb3f93d3e3c714
\ No newline at end of file
+d6dbf52d5f2f4cd72ae392ddef0bd8e3b98675bcb71f4a35901e10c0e4763cbe
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index e14d973..c155b4c 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-0f1edc1e24b9c517ad6645d922904b7738f01f405050f6f42edc7670ba528b18
\ No newline at end of file
+903f877bccb673d258b52a95e282bada0c0be1ffae62d30c21791c2d326c8c0a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index 519fb82..c44807a 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-6e408beca9d70e603620e6c91cd03a2ec753d8cb9b824edbd34916db9f95068c
\ No newline at end of file
+b2e1b540e6b3fe0ace20e8d1afaca2c44c93be317e32890bbfc8118f41c10c0a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 3e6587d..44f8f99 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-211d6e6c98a1d7d9bc2f17f700b6909a0a7158cb1b4775bb17f860d4e1b60ca0
\ No newline at end of file
+8f60074539dab1c95cca763c690b2afb8d4edc07810765d6439a9024330d9178
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index d47049c..14051f1 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-1cf8601dade7f3bc11087f23bf5719643dafb75f962e9164ed21717b26fca62f
\ No newline at end of file
+0158be74cc02831a535ebd72326a663c36e79764a99e19462d617f2f538934a4
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index 4a2d62a..4105aa7 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-7143d93a643dedbf59a54c553931f7dac9f218dd50ca8e3c185aa88564fde79e
\ No newline at end of file
+ebe10de8ffc95d9d0e3472db032a4eda5e21b6dd36fade5e97fca08745a70552
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 1042a8e..8710f42 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-63af94694153292f5adababaeb8e0efecadcb03352508dc38aa0f8d6081c4507
\ No newline at end of file
+db89febc96247d6b0730c7d21041cfef074256866e20aff8372207b962aacd14
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index fd09267..19b94ad 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-8f025f16146c9ce5f57054c20916a7a4af5289eeb2fe901888774bbccb6e626f
\ No newline at end of file
+381a0bab35e9e91a9746bd1b6611d773fa6c77aff0317764b9a0379762be16c9
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index 3a9889e..ee2f814 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-b33718cc3ccda4ae18c3979b093936998d7c048a1b645ee2b19d275ccee7fb59
\ No newline at end of file
+99c354c584dddbebe32f36005211dafce09847ea4b4c5003191eb585a1c81576
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 1f07525..0b9f807 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-af552654eb85d83844bc96ef9b2392f526bf3256f5563b0aef836833d0237b92
\ No newline at end of file
+296efa9843a3ec0fd016c5e3a14ea04d7f712988af01cc85909f625f76903fcb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index b840ab7..6388e4b 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-783302e483f9ebf95ee18f968caccdb679177edacceb832d81867e5a7ecc130d
\ No newline at end of file
+d3ad2e9bca5dcc8bb915938f0fcfd118ec1a38e4644a416898bc4b40ddfc7997
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 83df577..7be4f28 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-8e5561d64565aeab717e86f157242d209d50dfdcb7ec56a0b8c4896c503b02b3
\ No newline at end of file
+e4f2bd7e23727048719100b217afe9dd9e84873d9a0a863945a17bc0334057aa
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 2ebf08f..603b07b 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-baef1d30d83128da0244d404a131191a4dcc3800935e588944b1e20fd0be62d5
\ No newline at end of file
+29214222df2cb44b807621dbcef679d420a9ac70be06c62977effd09729b19ac
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 3df32b0..20b78d7 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-f075ed6e5aba4137398b9a2c02a0c577bd94f65d81f819c3a9aeb5648df81d34
\ No newline at end of file
+db33f28fb82d6cf8f7e2a856ba55e9e8ec21b515f8aafc9ebcd448f44611d74d
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 801d724..b48cf36 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-ad6be2e06d48d403d80d8fef2cea47f0e55e30e1b15ee44c960c5efeecd2bf0c
\ No newline at end of file
+5b067e82ffa03706fb9dd8248f02590086ff71c731d9c8d9d3d27363281b1e3c
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 1c169e8..d7ff9a3 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-a26c258774df481d0a8d6919eaa90fd4cf3c3a55b84dce5216d57f80886b101a
\ No newline at end of file
+1631a75b1f5a4648582a73a2786c7fc9e039eb4ed0159ea95c84560d586f7aa9
\ No newline at end of file