You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/08/04 06:52:48 UTC
[buildstream] branch tristan/light-cache-keys-without-sandbox-run created (now 170ca059b)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch tristan/light-cache-keys-without-sandbox-run
in repository https://gitbox.apache.org/repos/asf/buildstream.git
at 170ca059b element.py: Ignore environment/sandbox config when not running sandbox
This branch includes the following new commits:
new 86cd1496a sandbox.py: Clarifications to the utility of Sandbox._disable_run()
new 186738cde compose element: setup BST_RUN_COMMANDS
new 170ca059b element.py: Ignore environment/sandbox config when not running sandbox
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/03: compose element: setup BST_RUN_COMMANDS
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/light-cache-keys-without-sandbox-run
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 186738cde5011fab2648cdfa42e36d06c10f3bde
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu Aug 4 15:40:28 2022 +0900
compose element: setup BST_RUN_COMMANDS
Advertise BST_RUN_COMMANDS as False if we're not going to integrate the
dependencies.
---
src/buildstream/plugins/elements/compose.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py
index 90446c89b..3884a4a94 100644
--- a/src/buildstream/plugins/elements/compose.py
+++ b/src/buildstream/plugins/elements/compose.py
@@ -66,6 +66,10 @@ class ComposeElement(Element):
self.exclude = node.get_str_list("exclude")
self.include_orphans = node.get_bool("include-orphans")
+ # Inform the core that we will not need to run any commands in the sandbox
+ if not self.integration:
+ self.BST_RUN_COMMANDS = False
+
def preflight(self):
pass
[buildstream] 03/03: element.py: Ignore environment/sandbox config when not running sandbox
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/light-cache-keys-without-sandbox-run
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 170ca059b34054f319ec94eda636bf2d6df49f26
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu Aug 4 15:46:24 2022 +0900
element.py: Ignore environment/sandbox config when not running sandbox
When calculating the cache key, we can ignore sandbox related materials
when the plugin has specified BST_RUN_COMMANDS as False.
Fixes #1687
This patch also fixes cache key and completions tests which rely
on stable cache keys.
---
src/buildstream/element.py | 13 +++++++------
tests/cachekey/project/elements/compose1.expected | 2 +-
tests/cachekey/project/elements/compose2.expected | 2 +-
tests/cachekey/project/elements/compose3.expected | 2 +-
tests/cachekey/project/elements/compose4.expected | 2 +-
tests/cachekey/project/elements/compose5.expected | 2 +-
tests/cachekey/project/elements/import1.expected | 2 +-
tests/cachekey/project/elements/import2.expected | 2 +-
tests/cachekey/project/elements/import3.expected | 2 +-
tests/cachekey/project/elements/script1.expected | 2 +-
tests/cachekey/project/sources/local1.expected | 2 +-
tests/cachekey/project/sources/local2.expected | 2 +-
tests/cachekey/project/sources/remote1.expected | 2 +-
tests/cachekey/project/sources/remote2.expected | 2 +-
tests/cachekey/project/sources/tar1.expected | 2 +-
tests/cachekey/project/sources/tar2.expected | 2 +-
tests/cachekey/project/target.expected | 2 +-
tests/frontend/completions.py | 4 ++--
18 files changed, 25 insertions(+), 24 deletions(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 9a2b1bc2f..d96d22276 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -2209,9 +2209,6 @@ class Element(Plugin):
# Generate dict that is used as base for all cache keys
if self.__cache_key_dict is None:
- # Filter out nocache variables from the element's environment
- cache_env = {key: value for key, value in self.__environment.items() if key not in self.__env_nocache}
-
project = self._get_project()
self.__cache_key_dict = {
@@ -2220,15 +2217,19 @@ class Element(Plugin):
"element-plugin-key": self.get_unique_key(),
"element-plugin-name": self.get_kind(),
"element-plugin-version": self.BST_ARTIFACT_VERSION,
- "sandbox": self.__sandbox_config.to_dict(),
- "environment": cache_env,
"public": self.__public.strip_node_info(),
}
self.__cache_key_dict["sources"] = self.__sources.get_unique_key()
-
self.__cache_key_dict["fatal-warnings"] = sorted(project._fatal_warnings)
+ # Calculate sandbox related factors if this element runs the sandbox at assemble time.
+ if self.BST_RUN_COMMANDS:
+ # Filter out nocache variables from the element's environment
+ cache_env = {key: value for key, value in self.__environment.items() if key not in self.__env_nocache}
+ self.__cache_key_dict["sandbox"] = self.__sandbox_config.to_dict()
+ self.__cache_key_dict["environment"] = cache_env
+
cache_key_dict = self.__cache_key_dict.copy()
cache_key_dict["dependencies"] = dependencies
if weak_cache_key is not None:
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index 8f769e5a3..35987acee 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-79741eae7ac50457a67f06d2913f549080739dada5033b3015fb68b494536c5a
\ No newline at end of file
+d988151c1dace281305f6ca7933eb41dda658fc782220057d451e5151f4bff5a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index 1d1db3916..af8bb27ad 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-329f4ff2c3855f541ff2f85d0ba5ad4a87c73610914f84483c8462b095a8007e
\ No newline at end of file
+e5fd47fa23897007cb341e36b2a5466d29c17e49bbee84dcf2096a086ebbdac5
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index d56d56743..a03e50bbb 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-db8429d5f2a87335cfb3ca005b13a12267b3ba4ef369c72ee2d6dd237c4cf12a
\ No newline at end of file
+5faf568f052b3228f516a453c3030eaaac3ab0fd9a1328cb9ed2bae4a47dc556
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index f7c85c155..682ac721f 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-fff3c6f147a40ba9b47a4f6b39fc307934db891659d3d06258ab4b2458a5650f
\ No newline at end of file
+36f1d380ed1f0a7bbff7293ebccf2d8576d3ed72c0d7972647380fe1fa99c558
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 57ddda2b4..0d197358c 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-7040e4b5a4b82f2c4c511e62a125239dbb07ac171673ba167ab9e39d4e491763
\ No newline at end of file
+3288288fdb32362f0e31693ec409716d6b2bb9a4e3e8dbbeeb5825563975847f
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index a3133ccf6..c8c46da46 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-3b62fa381e9183353cac12f8e6ced23ea42931ed4f89ffe24625b6ec9f236ab5
\ No newline at end of file
+e2cfd9de891398c17f12197b9d2dfe94ed6549de9fadf224ec4b9f406e5f2031
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 4d32ff091..865ebd351 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-abeff9a432127674cff36acb36f45cd2665eebf75b589693db6676885e37c59c
\ No newline at end of file
+06f02cbfa4fa5d72026827847e66bf5e3c6d30944787f88631be50b20902c2c9
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 0fc1c1f9a..3b400892e 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-f2c5ae1c73a0d8abfbffb36b09bd74929e9b680bdb548f5faa1213a4c0423fe8
\ No newline at end of file
+e8d743e221d0e869714fb12770754998d3510cd0fbd6cde4dd789feeee383782
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 2eeb3e0f8..7c22cbeaa 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-eb81c0dfd2432b1ebf0bb052d6fa3a529438614c00210cfdc276bbcc2f3045d5
\ No newline at end of file
+e3c1c920d3b471d787c8d973000c82c8343c52dd199672a4d72ea6df1f9aa7e1
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 44f8f9915..cb41a9472 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-8f60074539dab1c95cca763c690b2afb8d4edc07810765d6439a9024330d9178
\ No newline at end of file
+a634f2b3ac93efda9e6698283d69cfa43520cda4cc3d4ef7ca2313da67c01f59
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index 14051f1a5..133d8a8ec 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-0158be74cc02831a535ebd72326a663c36e79764a99e19462d617f2f538934a4
\ No newline at end of file
+1f32c7ba211fa660cb84ef7dc85abfdb417a199eb3f99574650b3d9e0a009c5c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 0b9f80703..b8ae3b280 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-296efa9843a3ec0fd016c5e3a14ea04d7f712988af01cc85909f625f76903fcb
\ No newline at end of file
+f9ac5d3d38d30c1ce921de3a6f094e2732d0e82533f59338edb9d11854921462
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 6388e4b31..19014f2b2 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-d3ad2e9bca5dcc8bb915938f0fcfd118ec1a38e4644a416898bc4b40ddfc7997
\ No newline at end of file
+24971a99aa5a88f92bc99e3e0497b8b9603c06fd5a676e8d887a01d21d9d6abb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 7be4f284d..4bb7bafa4 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-e4f2bd7e23727048719100b217afe9dd9e84873d9a0a863945a17bc0334057aa
\ No newline at end of file
+ff706f0713c9399e00bfb39518c8af3887c03d022d2eca076b7dfeb47cb1d036
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 603b07be7..1d1fac3d6 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-29214222df2cb44b807621dbcef679d420a9ac70be06c62977effd09729b19ac
\ No newline at end of file
+1d8c2a1eca7460f3d925207ee1a44d24bccbccc1e4851bda924c502d1e8e4c4d
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 524b42a74..943c55ea0 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-94d8bb8fa9545152d59e4b9e141feb2c6190576457350cc6335186b024e99967
\ No newline at end of file
+6b7b400c0b70a2867fc80537428c25c6b4c9d2cdc69cd6273c06420f8248b292
\ No newline at end of file
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index ae289b5fd..22a4aaa12 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -340,8 +340,8 @@ def test_argument_artifact(cli, datafiles):
# Use hard coded artifact names, cache keys should be stable now
artifacts = [
- "test/import-bin/cb0c8c2e1881b09338aa3f533d224f83f06bdf263523d04ee197232c74f09357",
- "test/import-bin/edcfeda7d52c6bb77e632e31bd8ba40122125b2f50553b57c34947aa5fa709df",
+ "test/import-bin/2d14f24a0ab37ef86d8486140492cd75e9aee02f400b065548029643e4b13801",
+ "test/import-bin/7e01ad96507e126a81847c21528bbf8d87c7a9f6a251e8a7e0a6b71e38edf315",
]
# Test autocompletion of the artifact
[buildstream] 01/03: sandbox.py: Clarifications to the utility of Sandbox._disable_run()
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/light-cache-keys-without-sandbox-run
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 86cd1496a7b24e47b307416cc48f785b8e334c25
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu Aug 4 15:20:26 2022 +0900
sandbox.py: Clarifications to the utility of Sandbox._disable_run()
In the past, BuildStream had internal sandbox implementations which
would require staging files on the local filesystem in order to run,
but now we use buildbox-run exclusively and always use CasBasedDirectory.
The _use_cas_based_directory() method was removed in the commit
b487a11f6ee529c9cba505106d054c92c16e864c, and this was actually the
only reason why we had the Element.BST_RUN_COMMANDS flag at the time.
We still want BST_RUN_COMMANDS for other reasons, and it is useful
to keep the Sandbox._disable_run() function, but it's only purpose
now is to ensure the invariant that plugins who set BST_RUN_COMMANDS
to False do not actually run commands.
---
src/buildstream/element.py | 9 +++++----
src/buildstream/sandbox/sandbox.py | 17 +++++++++++++----
2 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 8f47a5b13..9a2b1bc2f 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1677,11 +1677,12 @@ class Element(Plugin):
rootdir, output_file, output_file, self.__sandbox_config
) as sandbox: # noqa
+ # Ensure that the plugin does not run commands if it said that it wouldn't
+ #
+ # We only disable commands here in _assemble() instead of __sandbox() because
+ # the user might still run a shell on an element which itself does not run commands.
+ #
if not self.BST_RUN_COMMANDS:
- # Element doesn't need to run any commands in the sandbox.
- #
- # Disable Sandbox.run() to allow CasBasedDirectory for all
- # sandboxes.
sandbox._disable_run()
# By default, the dynamic public data is the same as the static public data.
diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py
index e30414bc1..a7cf9c777 100644
--- a/src/buildstream/sandbox/sandbox.py
+++ b/src/buildstream/sandbox/sandbox.py
@@ -64,6 +64,13 @@ class SandboxCommandError(SandboxError):
self.collect = collect
+# An internal exception which can be used to explicitly trigger a bug / exception
+# which will be reported with a stack trace instead of reporting a user facing error
+#
+class _SandboxBug(Exception):
+ pass
+
+
class Sandbox:
"""Sandbox()
@@ -345,7 +352,7 @@ class Sandbox:
label: str = None
) -> Optional[int]:
if not self.__allow_run:
- raise SandboxError("Sandbox.run() has been disabled")
+ raise _SandboxBug("Element specified BST_RUN_COMMANDS as False but called Sandbox.run()")
# Fallback to the sandbox default settings for
# the cwd and env.
@@ -553,9 +560,11 @@ class Sandbox:
# _disable_run()
#
- # Raise exception if `Sandbox.run()` is called. This enables use of
- # CasBasedDirectory for faster staging when command execution is not
- # required.
+ # Raise exception if `Sandbox.run()` is called.
+ #
+ # This enforces an invariant by raising an exception if an element
+ # plugin ever set BST_RUN_COMMANDS to False but then proceeded to
+ # attempt to run the sandbox at assemble time.
#
def _disable_run(self):
self.__allow_run = False