You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/02/22 07:59:33 UTC
[buildstream] 01/01: element.py: Include weak cache key in cache key calculation.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/include-weak-key-in-cache-key
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4775e9be332a6b11a294c26446fd202100c01971
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Feb 22 16:55:14 2022 +0900
element.py: Include weak cache key in cache key calculation.
This should eliminate a class of issues related to strict/non-strict
discrepency.
This also updates the cache key test.
Fixes #1270
---
src/buildstream/element.py | 55 +++++++++++++----------
tests/cachekey/project/elements/build1.expected | 2 +-
tests/cachekey/project/elements/build2.expected | 2 +-
tests/cachekey/project/elements/build3.expected | 2 +-
tests/cachekey/project/elements/compose1.expected | 2 +-
tests/cachekey/project/elements/compose2.expected | 2 +-
tests/cachekey/project/elements/compose3.expected | 2 +-
tests/cachekey/project/elements/compose4.expected | 2 +-
tests/cachekey/project/elements/compose5.expected | 2 +-
tests/cachekey/project/elements/import1.expected | 2 +-
tests/cachekey/project/elements/import2.expected | 2 +-
tests/cachekey/project/elements/import3.expected | 2 +-
tests/cachekey/project/elements/script1.expected | 2 +-
tests/cachekey/project/sources/bzr1.expected | 2 +-
tests/cachekey/project/sources/git1.expected | 2 +-
tests/cachekey/project/sources/git2.expected | 2 +-
tests/cachekey/project/sources/git3.expected | 2 +-
tests/cachekey/project/sources/local1.expected | 2 +-
tests/cachekey/project/sources/local2.expected | 2 +-
tests/cachekey/project/sources/patch1.expected | 2 +-
tests/cachekey/project/sources/patch2.expected | 2 +-
tests/cachekey/project/sources/patch3.expected | 2 +-
tests/cachekey/project/sources/pip1.expected | 2 +-
tests/cachekey/project/sources/remote1.expected | 2 +-
tests/cachekey/project/sources/remote2.expected | 2 +-
tests/cachekey/project/sources/tar1.expected | 2 +-
tests/cachekey/project/sources/tar2.expected | 2 +-
tests/cachekey/project/sources/zip1.expected | 2 +-
tests/cachekey/project/sources/zip2.expected | 2 +-
tests/cachekey/project/target.expected | 2 +-
30 files changed, 61 insertions(+), 52 deletions(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index ee7e76c..e891349 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -2200,13 +2200,15 @@ class Element(Plugin):
# Args:
# dependencies (List[List[str]]): list of dependencies with project name,
# element name and optional cache key
+ # weak_cache_key (Optional[str]): the weak cache key, required for calculating the
+ # strict and strong cache keys
#
# Returns:
# (str): A hex digest cache key for this Element, or None
#
# None is returned if information for the cache key is missing.
#
- def _calculate_cache_key(self, dependencies):
+ def _calculate_cache_key(self, dependencies, weak_cache_key=None):
# No cache keys for dependencies which have no cache keys
if any(not all(dep) for dep in dependencies):
return None
@@ -2235,6 +2237,8 @@ class Element(Plugin):
cache_key_dict = self.__cache_key_dict.copy()
cache_key_dict["dependencies"] = dependencies
+ if weak_cache_key is not None:
+ cache_key_dict["weak-cache-key"] = weak_cache_key
return _cachekey.generate_key(cache_key_dict)
@@ -3219,37 +3223,40 @@ class Element(Plugin):
# Tracking may still be pending
return
+ # Calculate weak cache key first, as it is required for generating the other keys.
+ #
+ # This code can be run multiple times until the strict key can be calculated,
+ # so let's ensure we only ever calculate the weak key once, even though we need
+ # to resolve it before we can resolve the strict key.
+ if self.__weak_cache_key is None:
+ # Weak cache key includes names of direct build dependencies
+ # so as to only trigger rebuilds when the shape of the
+ # dependencies change.
+ #
+ # Some conditions cause dependencies to be strict, such
+ # that this element will be rebuilt anyway if the dependency
+ # changes even in non strict mode, for these cases we just
+ # encode the dependency's weak cache key instead of it's name.
+ #
+ dependencies = [
+ [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)]
+ if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
+ else [e.project_name, e.name]
+ for e in self._dependencies(_Scope.BUILD)
+ ]
+ self.__weak_cache_key = self._calculate_cache_key(dependencies)
+
context = self._get_context()
# Calculate the strict cache key
dependencies = [[e.project_name, e.name, e.__strict_cache_key] for e in self._dependencies(_Scope.BUILD)]
- self.__strict_cache_key = self._calculate_cache_key(dependencies)
+ self.__strict_cache_key = self._calculate_cache_key(dependencies, self.__weak_cache_key)
if self.__strict_cache_key is None:
# Cache keys cannot be calculated yet as a build dependency doesn't
# have a cache key yet.
return
- # Calculate weak cache key
- #
- # Weak cache key includes names of direct build dependencies
- # so as to only trigger rebuilds when the shape of the
- # dependencies change.
- #
- # Some conditions cause dependencies to be strict, such
- # that this element will be rebuilt anyway if the dependency
- # changes even in non strict mode, for these cases we just
- # encode the dependency's weak cache key instead of it's name.
- #
- dependencies = [
- [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)]
- if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
- else [e.project_name, e.name]
- for e in self._dependencies(_Scope.BUILD)
- ]
-
- self.__weak_cache_key = self._calculate_cache_key(dependencies)
-
# As the strict cache key has already been calculated, it should always
# be possible to calculate the weak cache key as well.
assert self.__weak_cache_key is not None
@@ -3287,9 +3294,11 @@ class Element(Plugin):
strong_key, _, _ = self.__artifact.get_metadata_keys()
self.__cache_key = strong_key
elif self.__assemble_scheduled or self.__assemble_done:
+ assert self.__weak_cache_key is not None
+
# Artifact will or has been built, not downloaded
dependencies = [[e.project_name, e.name, e._get_cache_key()] for e in self._dependencies(_Scope.BUILD)]
- self.__cache_key = self._calculate_cache_key(dependencies)
+ self.__cache_key = self._calculate_cache_key(dependencies, self.__weak_cache_key)
if self.__cache_key is None:
# Strong cache key could not be calculated yet
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index f9e425c..98311d4 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-df973166242fc7613903e09b39f8c44990868af0554421ee7c1beaf9fc268030
\ No newline at end of file
+7f00e0e1d2f7f154895c0392816750984ddcbcde290a87df475a3ff1ef151308
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index 9f0a4d2..f07a044 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-0d3d9cb2c3d8569806f4822a2bd52150a675d4621883b35e02f266cfc10bc457
\ No newline at end of file
+11578c1823d8fad0143bb1afd195a6dbc5b36a11834ab04d1babe7d5750b5341
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build3.expected b/tests/cachekey/project/elements/build3.expected
index fd339a4..2edc024 100644
--- a/tests/cachekey/project/elements/build3.expected
+++ b/tests/cachekey/project/elements/build3.expected
@@ -1 +1 @@
-8fbfdfed830260389b941583dcde6533202a3e970063aa6569dfd1b4dd303356
\ No newline at end of file
+bf6067aaba914c449080a8e477d8b206dd3a756085874be4ec0d902850d3f0a1
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index fd88e67..8f769e5 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-f3eb266781b7cea1230d5def8f49d21312ae98e2156b77ff429690ebefcf6d5a
\ No newline at end of file
+79741eae7ac50457a67f06d2913f549080739dada5033b3015fb68b494536c5a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index d979a89..1d1db39 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-827b5de4a4cc7cb2a0738d792287af56ca95e29da75aea5c7674cb6fa9d06b70
\ No newline at end of file
+329f4ff2c3855f541ff2f85d0ba5ad4a87c73610914f84483c8462b095a8007e
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 291e94c..d56d567 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-82e9499af43b8ce933e61b2b93f342930616ac06f2288ea8b00c9245935beb67
\ No newline at end of file
+db8429d5f2a87335cfb3ca005b13a12267b3ba4ef369c72ee2d6dd237c4cf12a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 388fe77..f7c85c1 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-77ff7b5a2286bce2dbe2b6bb7e98908d709c5b00450af5c8e9221eccd481eae2
\ No newline at end of file
+fff3c6f147a40ba9b47a4f6b39fc307934db891659d3d06258ab4b2458a5650f
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index d7625b0..57ddda2 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-6239ae1b6b33c6bdb3b25198c2d4a43d0c77dd7d95f540a42e139188a074dcfd
\ No newline at end of file
+7040e4b5a4b82f2c4c511e62a125239dbb07ac171673ba167ab9e39d4e491763
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index 5f013e6..a3133cc 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-2e13c26448868c7c30cf7e4549627d757cadab2901d578f40acdd17aaa535816
\ No newline at end of file
+3b62fa381e9183353cac12f8e6ced23ea42931ed4f89ffe24625b6ec9f236ab5
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 0ed5739..4d32ff0 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-4b2a646a93b955392449d8e9c4dcb121e52e1698a11859d90df12d08da8a4564
\ No newline at end of file
+abeff9a432127674cff36acb36f45cd2665eebf75b589693db6676885e37c59c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index f071fe2..0fc1c1f 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-71b8113e587bb9e6549bd95b71dac7b5d2c7d7d2de59f3f27c247fec860a55b5
\ No newline at end of file
+f2c5ae1c73a0d8abfbffb36b09bd74929e9b680bdb548f5faa1213a4c0423fe8
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index f7d213b..2eeb3e0 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-4b5a4267ba32fca936acf7212213e03c5a6f0448f66444f54009243b5425ca3c
\ No newline at end of file
+eb81c0dfd2432b1ebf0bb052d6fa3a529438614c00210cfdc276bbcc2f3045d5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 3a344db..dcad774 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-a19ed9f67c1106b3af0e43dd2e4485b7cc0ef40a26c39f76ae102a2791dac0b0
\ No newline at end of file
+a8ca1b3d852de71c0d97fbdddfe86f815a8edc62acf5aebbdcca1972a1627b12
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 66594a0..7cb0871 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-d554f77e28b68c4085603e67dc3db69bdedb7284efceee60f7bb3f93d3e3c714
\ No newline at end of file
+d6dbf52d5f2f4cd72ae392ddef0bd8e3b98675bcb71f4a35901e10c0e4763cbe
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index e14d973..c155b4c 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-0f1edc1e24b9c517ad6645d922904b7738f01f405050f6f42edc7670ba528b18
\ No newline at end of file
+903f877bccb673d258b52a95e282bada0c0be1ffae62d30c21791c2d326c8c0a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index 519fb82..c44807a 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-6e408beca9d70e603620e6c91cd03a2ec753d8cb9b824edbd34916db9f95068c
\ No newline at end of file
+b2e1b540e6b3fe0ace20e8d1afaca2c44c93be317e32890bbfc8118f41c10c0a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 3e6587d..44f8f99 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-211d6e6c98a1d7d9bc2f17f700b6909a0a7158cb1b4775bb17f860d4e1b60ca0
\ No newline at end of file
+8f60074539dab1c95cca763c690b2afb8d4edc07810765d6439a9024330d9178
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index d47049c..14051f1 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-1cf8601dade7f3bc11087f23bf5719643dafb75f962e9164ed21717b26fca62f
\ No newline at end of file
+0158be74cc02831a535ebd72326a663c36e79764a99e19462d617f2f538934a4
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index 4a2d62a..4105aa7 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-7143d93a643dedbf59a54c553931f7dac9f218dd50ca8e3c185aa88564fde79e
\ No newline at end of file
+ebe10de8ffc95d9d0e3472db032a4eda5e21b6dd36fade5e97fca08745a70552
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 1042a8e..8710f42 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-63af94694153292f5adababaeb8e0efecadcb03352508dc38aa0f8d6081c4507
\ No newline at end of file
+db89febc96247d6b0730c7d21041cfef074256866e20aff8372207b962aacd14
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index fd09267..19b94ad 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-8f025f16146c9ce5f57054c20916a7a4af5289eeb2fe901888774bbccb6e626f
\ No newline at end of file
+381a0bab35e9e91a9746bd1b6611d773fa6c77aff0317764b9a0379762be16c9
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index 3a9889e..ee2f814 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-b33718cc3ccda4ae18c3979b093936998d7c048a1b645ee2b19d275ccee7fb59
\ No newline at end of file
+99c354c584dddbebe32f36005211dafce09847ea4b4c5003191eb585a1c81576
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 1f07525..0b9f807 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-af552654eb85d83844bc96ef9b2392f526bf3256f5563b0aef836833d0237b92
\ No newline at end of file
+296efa9843a3ec0fd016c5e3a14ea04d7f712988af01cc85909f625f76903fcb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index b840ab7..6388e4b 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-783302e483f9ebf95ee18f968caccdb679177edacceb832d81867e5a7ecc130d
\ No newline at end of file
+d3ad2e9bca5dcc8bb915938f0fcfd118ec1a38e4644a416898bc4b40ddfc7997
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 83df577..7be4f28 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-8e5561d64565aeab717e86f157242d209d50dfdcb7ec56a0b8c4896c503b02b3
\ No newline at end of file
+e4f2bd7e23727048719100b217afe9dd9e84873d9a0a863945a17bc0334057aa
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 2ebf08f..603b07b 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-baef1d30d83128da0244d404a131191a4dcc3800935e588944b1e20fd0be62d5
\ No newline at end of file
+29214222df2cb44b807621dbcef679d420a9ac70be06c62977effd09729b19ac
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 3df32b0..20b78d7 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-f075ed6e5aba4137398b9a2c02a0c577bd94f65d81f819c3a9aeb5648df81d34
\ No newline at end of file
+db33f28fb82d6cf8f7e2a856ba55e9e8ec21b515f8aafc9ebcd448f44611d74d
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 801d724..b48cf36 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-ad6be2e06d48d403d80d8fef2cea47f0e55e30e1b15ee44c960c5efeecd2bf0c
\ No newline at end of file
+5b067e82ffa03706fb9dd8248f02590086ff71c731d9c8d9d3d27363281b1e3c
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 1c169e8..d7ff9a3 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-a26c258774df481d0a8d6919eaa90fd4cf3c3a55b84dce5216d57f80886b101a
\ No newline at end of file
+1631a75b1f5a4648582a73a2786c7fc9e039eb4ed0159ea95c84560d586f7aa9
\ No newline at end of file