You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ju...@apache.org on 2022/07/08 18:56:24 UTC
[buildstream] branch juerg/opt created (now cbc8e791f)
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a change to branch juerg/opt
in repository https://gitbox.apache.org/repos/asf/buildstream.git
at cbc8e791f _elementsources.py: Use `StageTree` to stage previous sources
This branch includes the following new commits:
new caf965bea Clean the build directory after a successful build
new cbc8e791f _elementsources.py: Use `StageTree` to stage previous sources
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/02: Clean the build directory after a successful build
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/opt
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit caf965beaa02cd7b45874a453fdbb6eea8d2e19d
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu Jul 7 22:08:04 2022 +0200
Clean the build directory after a successful build
Capturing the build directory takes time and uses disk space of the CAS
cache. If the `cache-buildtrees` option is set to `auto` (the default)
or `never`, the contents of the build directory can be removed after a
successful build as the build tree is no longer needed, avoiding the
capture costs.
---
src/buildstream/buildelement.py | 5 ++++
src/buildstream/element.py | 5 ++--
src/buildstream/sandbox/_sandboxreapi.py | 7 +++++
src/buildstream/sandbox/sandbox.py | 47 ++++++++++++++++++++++++++++++++
4 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py
index ec508f59c..05b4f0d89 100644
--- a/src/buildstream/buildelement.py
+++ b/src/buildstream/buildelement.py
@@ -334,6 +334,11 @@ class BuildElement(Element):
for cmd in commands:
self.__run_command(sandbox, cmd)
+ if not self._get_workspace():
+ # Empty the build directory after successful build to avoid overhead of
+ # capturing the build directory (unless buildtree caching has been configured).
+ sandbox._clean_build_directory()
+
# Return the payload, this is configurable but is generally
# always the /buildstream-install directory
return self.get_variable("install-root")
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 0ca7f59b7..d73045d81 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1651,9 +1651,8 @@ class Element(Plugin):
# download when it's not needed.
buildroot = self.get_variable("build-root")
cache_buildtrees = context.cache_buildtrees
- if cache_buildtrees != _CacheBuildTrees.NEVER:
- always_cache_buildtrees = cache_buildtrees == _CacheBuildTrees.ALWAYS
- sandbox._set_build_directory(buildroot, always=always_cache_buildtrees)
+ always_cache_buildtrees = cache_buildtrees == _CacheBuildTrees.ALWAYS
+ sandbox._set_build_directory(buildroot, always=always_cache_buildtrees)
if not self.BST_RUN_COMMANDS:
# Element doesn't need to run any commands in the sandbox.
diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py
index 53f8b97a5..dbf06ac67 100644
--- a/src/buildstream/sandbox/_sandboxreapi.py
+++ b/src/buildstream/sandbox/_sandboxreapi.py
@@ -278,3 +278,10 @@ class _SandboxREAPIBatch(_SandboxBatch):
def create_empty_file(self, name):
self.script += "touch -- {}\n".format(shlex.quote(name))
+
+ def clean_directory(self, name):
+ # Do not treat error during cleanup as a fatal build error
+ self.script += "rm -rf -- {} || true\n".format(shlex.quote(name))
+ # Working directory may be a subdirectory of the build directory.
+ # Recreate it if necessary as output capture requires the working directory to exist.
+ self.script += "mkdir -p {} || true\n".format(shlex.quote(self.first_command.cwd))
diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py
index d788899f6..400ab7bc3 100644
--- a/src/buildstream/sandbox/sandbox.py
+++ b/src/buildstream/sandbox/sandbox.py
@@ -526,6 +526,32 @@ class Sandbox:
cwd_vdir = vdir.open_directory(cwd.lstrip(os.sep), create=True)
cwd_vdir._create_empty_file(name)
+ # _clean_build_directory()
+ #
+ # Remove the contents of the build directory after a successful build to
+ # avoid the capture costs (CPU, I/O and buildbox-casd disk usage).
+ #
+ # This will be skipped if the `cache_buildtrees` configuration is set to `always`.
+ #
+ def _clean_build_directory(self):
+ assert self._build_directory
+
+ # Preserve build directory if it is to be stored in the artifact even for a successful build
+ if self._build_directory_always:
+ return
+
+ if self.__batch:
+ batch_clean = _SandboxBatchCleanDirectory(self._build_directory)
+
+ current_group = self.__batch.current_group
+ current_group.append(batch_clean)
+ else:
+ vdir = self.get_virtual_directory()
+ relative_path = self._build_directory.lstrip(os.sep)
+ if vdir.exists(relative_path):
+ vdir.remove(relative_path, recursive=True)
+ vdir.open_directory(relative_path, create=True)
+
# _get_element_name()
#
# Get the plugin's element full name
@@ -642,6 +668,13 @@ class _SandboxBatch:
cwd_vdir = vdir.open_directory(cwd.lstrip(os.sep), create=True)
cwd_vdir._create_empty_file(name)
+ def clean_directory(self, name):
+ vdir = self.sandbox.get_virtual_directory()
+ relative_path = name.lstrip(os.sep)
+ if vdir.exists(relative_path):
+ vdir.remove(relative_path, recursive=True)
+ vdir.open_directory(relative_path, create=True)
+
# _SandboxBatchItem()
#
@@ -707,3 +740,17 @@ class _SandboxBatchFile(_SandboxBatchItem):
def execute(self, batch):
batch.create_empty_file(self.name)
+
+
+# _SandboxBatchCleanDirectory()
+#
+# A directory cleaning item in a command batch.
+#
+class _SandboxBatchCleanDirectory(_SandboxBatchItem):
+ def __init__(self, name):
+ super().__init__()
+
+ self.name = name
+
+ def execute(self, batch):
+ batch.clean_directory(self.name)
[buildstream] 02/02: _elementsources.py: Use `StageTree` to stage previous sources
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/opt
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cbc8e791fc93b1f4028996595856fc0a670c9244
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Fri Jul 8 14:32:09 2022 +0200
_elementsources.py: Use `StageTree` to stage previous sources
This will use buildbox-fuse (via buildbox-casd) if available, improving
performance of both stage and capture operations.
---
src/buildstream/_cas/cascache.py | 29 +++++++++++++++++++++++++++++
src/buildstream/_elementsources.py | 10 +++++-----
2 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index c6a58098f..6ac3da557 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -443,6 +443,35 @@ class CASCache:
return utils._message_digest(root_directory)
+ @contextlib.contextmanager
+ def stage_directory(self, directory_digest):
+ local_cas = self.get_local_cas()
+
+ request = local_cas_pb2.StageTreeRequest()
+ request.root_digest.CopyFrom(directory_digest)
+
+ done_event = threading.Event()
+
+ def request_iterator():
+ yield request
+
+ # Wait until staged tree is no longer needed
+ done_event.wait()
+
+ # A second (empty) request indicates that the staging location can be cleaned up
+ yield local_cas_pb2.StageTreeRequest()
+
+ response_stream = local_cas.StageTree(request_iterator())
+
+ # Read primary response and yield staging location
+ response = next(response_stream)
+ yield response.path
+
+ # Staged tree is no longer needed
+ done_event.set()
+ # Wait for cleanup to complete
+ next(response_stream)
+
# missing_blobs_for_directory():
#
# Determine which blobs of a directory tree are missing on the remote.
diff --git a/src/buildstream/_elementsources.py b/src/buildstream/_elementsources.py
index f089d17b1..315e973c0 100644
--- a/src/buildstream/_elementsources.py
+++ b/src/buildstream/_elementsources.py
@@ -459,7 +459,8 @@ class ElementSources:
# stop (Source): Only stage sources listed before this source
#
def _stage(self, *, stop=None):
- vdir = CasBasedDirectory(self._context.get_cascache())
+ cas = self._context.get_cascache()
+ vdir = CasBasedDirectory(cas)
for source in self._sources:
if source == stop:
@@ -474,10 +475,9 @@ class ElementSources:
if source.BST_STAGE_VIRTUAL_DIRECTORY:
source._stage(vsubdir)
else:
- with utils._tempdir(dir=self._context.tmpdir, prefix="staging-temp") as tmpdir:
- # Stage previous sources
- vsubdir._export_files(tmpdir)
-
+ # Stage previous sources
+ with cas.stage_directory(vsubdir._get_digest()) as tmpdir:
+ # Stage current source
source._stage(tmpdir)
# Capture modified tree