You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ju...@apache.org on 2022/07/08 06:00:37 UTC
[buildstream] branch juerg/clean-build-directory created (now 1232bbe5c)
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a change to branch juerg/clean-build-directory
in repository https://gitbox.apache.org/repos/asf/buildstream.git
at 1232bbe5c Clean the build directory after a successful build
This branch includes the following new commits:
new 1232bbe5c Clean the build directory after a successful build
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: Clean the build directory after a successful build
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/clean-build-directory
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1232bbe5ca4b27ec025943719572e0bce5f05f28
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu Jul 7 22:08:04 2022 +0200
Clean the build directory after a successful build
Capturing the build directory takes time and uses disk space of the CAS
cache. If the `cache-buildtrees` option is set to `auto` (the default)
or `never`, the contents of the build directory can be removed after a
successful build as the build tree is no longer needed, avoiding the
capture costs.
---
src/buildstream/buildelement.py | 5 ++++
src/buildstream/element.py | 5 ++--
src/buildstream/sandbox/_sandboxreapi.py | 7 +++++
src/buildstream/sandbox/sandbox.py | 47 ++++++++++++++++++++++++++++++++
4 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py
index f9dc36fae..51952e0ee 100644
--- a/src/buildstream/buildelement.py
+++ b/src/buildstream/buildelement.py
@@ -338,6 +338,11 @@ class BuildElement(Element):
for cmd in commands:
self.__run_command(sandbox, cmd)
+ if not self._get_workspace():
+ # Empty the build directory after successful build to avoid overhead of
+ # capturing the build directory (unless buildtree caching has been configured).
+ sandbox._clean_build_directory()
+
# Return the payload, this is configurable but is generally
# always the /buildstream-install directory
return self.get_variable("install-root")
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 0ca7f59b7..d73045d81 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1651,9 +1651,8 @@ class Element(Plugin):
# download when it's not needed.
buildroot = self.get_variable("build-root")
cache_buildtrees = context.cache_buildtrees
- if cache_buildtrees != _CacheBuildTrees.NEVER:
- always_cache_buildtrees = cache_buildtrees == _CacheBuildTrees.ALWAYS
- sandbox._set_build_directory(buildroot, always=always_cache_buildtrees)
+ always_cache_buildtrees = cache_buildtrees == _CacheBuildTrees.ALWAYS
+ sandbox._set_build_directory(buildroot, always=always_cache_buildtrees)
if not self.BST_RUN_COMMANDS:
# Element doesn't need to run any commands in the sandbox.
diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py
index 53f8b97a5..dbf06ac67 100644
--- a/src/buildstream/sandbox/_sandboxreapi.py
+++ b/src/buildstream/sandbox/_sandboxreapi.py
@@ -278,3 +278,10 @@ class _SandboxREAPIBatch(_SandboxBatch):
def create_empty_file(self, name):
self.script += "touch -- {}\n".format(shlex.quote(name))
+
+ def clean_directory(self, name):
+ # Do not treat error during cleanup as a fatal build error
+ self.script += "rm -rf -- {} || true\n".format(shlex.quote(name))
+ # Working directory may be a subdirectory of the build directory.
+ # Recreate it if necessary as output capture requires the working directory to exist.
+ self.script += "mkdir -p {} || true\n".format(shlex.quote(self.first_command.cwd))
diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py
index d788899f6..400ab7bc3 100644
--- a/src/buildstream/sandbox/sandbox.py
+++ b/src/buildstream/sandbox/sandbox.py
@@ -526,6 +526,32 @@ class Sandbox:
cwd_vdir = vdir.open_directory(cwd.lstrip(os.sep), create=True)
cwd_vdir._create_empty_file(name)
+ # _clean_build_directory()
+ #
+ # Remove the contents of the build directory after a successful build to
+ # avoid the capture costs (CPU, I/O and buildbox-casd disk usage).
+ #
+ # This will be skipped if the `cache_buildtrees` configuration is set to `always`.
+ #
+ def _clean_build_directory(self):
+ assert self._build_directory
+
+ # Preserve build directory if it is to be stored in the artifact even for a successful build
+ if self._build_directory_always:
+ return
+
+ if self.__batch:
+ batch_clean = _SandboxBatchCleanDirectory(self._build_directory)
+
+ current_group = self.__batch.current_group
+ current_group.append(batch_clean)
+ else:
+ vdir = self.get_virtual_directory()
+ relative_path = self._build_directory.lstrip(os.sep)
+ if vdir.exists(relative_path):
+ vdir.remove(relative_path, recursive=True)
+ vdir.open_directory(relative_path, create=True)
+
# _get_element_name()
#
# Get the plugin's element full name
@@ -642,6 +668,13 @@ class _SandboxBatch:
cwd_vdir = vdir.open_directory(cwd.lstrip(os.sep), create=True)
cwd_vdir._create_empty_file(name)
+ def clean_directory(self, name):
+ vdir = self.sandbox.get_virtual_directory()
+ relative_path = name.lstrip(os.sep)
+ if vdir.exists(relative_path):
+ vdir.remove(relative_path, recursive=True)
+ vdir.open_directory(relative_path, create=True)
+
# _SandboxBatchItem()
#
@@ -707,3 +740,17 @@ class _SandboxBatchFile(_SandboxBatchItem):
def execute(self, batch):
batch.create_empty_file(self.name)
+
+
+# _SandboxBatchCleanDirectory()
+#
+# A directory cleaning item in a command batch.
+#
+class _SandboxBatchCleanDirectory(_SandboxBatchItem):
+ def __init__(self, name):
+ super().__init__()
+
+ self.name = name
+
+ def execute(self, batch):
+ batch.clean_directory(self.name)