[buildstream] 01/01: Clean the build directory after a successful build

[ju...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

juergbi pushed a commit to branch juerg/clean-build-directory
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 1232bbe5ca4b27ec025943719572e0bce5f05f28
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu Jul 7 22:08:04 2022 +0200

    Clean the build directory after a successful build
    
    Capturing the build directory takes time and uses disk space of the CAS
    cache. If the `cache-buildtrees` option is set to `auto` (the default)
    or `never`, the contents of the build directory can be removed after a
    successful build as the build tree is no longer needed, avoiding the
    capture costs.
---
 src/buildstream/buildelement.py          |  5 ++++
 src/buildstream/element.py               |  5 ++--
 src/buildstream/sandbox/_sandboxreapi.py |  7 +++++
 src/buildstream/sandbox/sandbox.py       | 47 ++++++++++++++++++++++++++++++++
 4 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py
index f9dc36fae..51952e0ee 100644
--- a/src/buildstream/buildelement.py
+++ b/src/buildstream/buildelement.py
@@ -338,6 +338,11 @@ class BuildElement(Element):
                 for cmd in commands:
                     self.__run_command(sandbox, cmd)
 
+            if not self._get_workspace():
+                # Empty the build directory after successful build to avoid overhead of
+                # capturing the build directory (unless buildtree caching has been configured).
+                sandbox._clean_build_directory()
+
             # Return the payload, this is configurable but is generally
             # always the /buildstream-install directory
             return self.get_variable("install-root")
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 0ca7f59b7..d73045d81 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1651,9 +1651,8 @@ class Element(Plugin):
                 # download when it's not needed.
                 buildroot = self.get_variable("build-root")
                 cache_buildtrees = context.cache_buildtrees
-                if cache_buildtrees != _CacheBuildTrees.NEVER:
-                    always_cache_buildtrees = cache_buildtrees == _CacheBuildTrees.ALWAYS
-                    sandbox._set_build_directory(buildroot, always=always_cache_buildtrees)
+                always_cache_buildtrees = cache_buildtrees == _CacheBuildTrees.ALWAYS
+                sandbox._set_build_directory(buildroot, always=always_cache_buildtrees)
 
                 if not self.BST_RUN_COMMANDS:
                     # Element doesn't need to run any commands in the sandbox.
diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py
index 53f8b97a5..dbf06ac67 100644
--- a/src/buildstream/sandbox/_sandboxreapi.py
+++ b/src/buildstream/sandbox/_sandboxreapi.py
@@ -278,3 +278,10 @@ class _SandboxREAPIBatch(_SandboxBatch):
 
     def create_empty_file(self, name):
         self.script += "touch -- {}\n".format(shlex.quote(name))
+
+    def clean_directory(self, name):
+        # Do not treat error during cleanup as a fatal build error
+        self.script += "rm -rf -- {} || true\n".format(shlex.quote(name))
+        # Working directory may be a subdirectory of the build directory.
+        # Recreate it if necessary as output capture requires the working directory to exist.
+        self.script += "mkdir -p {} || true\n".format(shlex.quote(self.first_command.cwd))
diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py
index d788899f6..400ab7bc3 100644
--- a/src/buildstream/sandbox/sandbox.py
+++ b/src/buildstream/sandbox/sandbox.py
@@ -526,6 +526,32 @@ class Sandbox:
             cwd_vdir = vdir.open_directory(cwd.lstrip(os.sep), create=True)
             cwd_vdir._create_empty_file(name)
 
+    # _clean_build_directory()
+    #
+    # Remove the contents of the build directory after a successful build to
+    # avoid the capture costs (CPU, I/O and buildbox-casd disk usage).
+    #
+    # This will be skipped if the `cache_buildtrees` configuration is set to `always`.
+    #
+    def _clean_build_directory(self):
+        assert self._build_directory
+
+        # Preserve build directory if it is to be stored in the artifact even for a successful build
+        if self._build_directory_always:
+            return
+
+        if self.__batch:
+            batch_clean = _SandboxBatchCleanDirectory(self._build_directory)
+
+            current_group = self.__batch.current_group
+            current_group.append(batch_clean)
+        else:
+            vdir = self.get_virtual_directory()
+            relative_path = self._build_directory.lstrip(os.sep)
+            if vdir.exists(relative_path):
+                vdir.remove(relative_path, recursive=True)
+                vdir.open_directory(relative_path, create=True)
+
     # _get_element_name()
     #
     # Get the plugin's element full name
@@ -642,6 +668,13 @@ class _SandboxBatch:
         cwd_vdir = vdir.open_directory(cwd.lstrip(os.sep), create=True)
         cwd_vdir._create_empty_file(name)
 
+    def clean_directory(self, name):
+        vdir = self.sandbox.get_virtual_directory()
+        relative_path = name.lstrip(os.sep)
+        if vdir.exists(relative_path):
+            vdir.remove(relative_path, recursive=True)
+            vdir.open_directory(relative_path, create=True)
+
 
 # _SandboxBatchItem()
 #
@@ -707,3 +740,17 @@ class _SandboxBatchFile(_SandboxBatchItem):
 
     def execute(self, batch):
         batch.create_empty_file(self.name)
+
+
+# _SandboxBatchCleanDirectory()
+#
+# A directory cleaning item in a command batch.
+#
+class _SandboxBatchCleanDirectory(_SandboxBatchItem):
+    def __init__(self, name):
+        super().__init__()
+
+        self.name = name
+
+    def execute(self, batch):
+        batch.clean_directory(self.name)