You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:19:11 UTC
[buildstream] branch buildbox-pre-will created (now 5600890)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch buildbox-pre-will
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 5600890 HACK: _platform/linux.py: Use BuildBox sandbox backend
This branch includes the following new commits:
new 2c26bd9 WIP: sandbox: Add initial SandboxBuildBox
new 5600890 HACK: _platform/linux.py: Use BuildBox sandbox backend
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/02: HACK: _platform/linux.py: Use BuildBox sandbox
backend
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch buildbox-pre-will
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5600890b5c51b0a2cc4945c6bfa8ecee59514a26
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Wed Nov 14 10:28:46 2018 +0100
HACK: _platform/linux.py: Use BuildBox sandbox backend
---
buildstream/_platform/linux.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index e4ce025..b8e2c38 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -22,7 +22,7 @@ import subprocess
from .. import _site
from .. import utils
-from ..sandbox import SandboxDummy
+from ..sandbox import SandboxDummy, SandboxBuildBox
from .platform import Platform
from .._exceptions import PlatformError
@@ -66,7 +66,8 @@ class Linux(Platform):
if not self._local_sandbox_available:
return self._create_dummy_sandbox(*args, **kwargs)
else:
- return self._create_bwrap_sandbox(*args, **kwargs)
+ # return self._create_bwrap_sandbox(*args, **kwargs)
+ return SandboxBuildBox(*args, **kwargs)
def check_sandbox_config(self, config):
if not self._local_sandbox_available:
[buildstream] 01/02: WIP: sandbox: Add initial SandboxBuildBox
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch buildbox-pre-will
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2c26bd9099ef9775d62481ff02b18bb8c26e573b
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Wed Aug 8 14:35:25 2018 +0200
WIP: sandbox: Add initial SandboxBuildBox
---
buildstream/sandbox/__init__.py | 1 +
buildstream/sandbox/_sandboxbuildbox.py | 198 ++++++++++++++++++++++++++++++++
2 files changed, 199 insertions(+)
diff --git a/buildstream/sandbox/__init__.py b/buildstream/sandbox/__init__.py
index 5966d19..5a7b412 100644
--- a/buildstream/sandbox/__init__.py
+++ b/buildstream/sandbox/__init__.py
@@ -20,3 +20,4 @@
from .sandbox import Sandbox, SandboxFlags, SandboxCommandError
from ._sandboxremote import SandboxRemote
from ._sandboxdummy import SandboxDummy
+from ._sandboxbuildbox import SandboxBuildBox
diff --git a/buildstream/sandbox/_sandboxbuildbox.py b/buildstream/sandbox/_sandboxbuildbox.py
new file mode 100644
index 0000000..8795930
--- /dev/null
+++ b/buildstream/sandbox/_sandboxbuildbox.py
@@ -0,0 +1,198 @@
+#
+# Copyright (C) 2018 Bloomberg LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import signal
+import subprocess
+from contextlib import ExitStack
+
+import psutil
+
+from .. import utils, _signals
+from . import Sandbox, SandboxFlags
+from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from ..storage._casbaseddirectory import CasBasedDirectory
+
+
+# SandboxBuidBox()
+#
+# BuildBox-based sandbox implementation.
+#
+class SandboxBuildBox(Sandbox):
+
+ def _run(self, command, flags, *, cwd, env):
+ stdout, stderr = self._get_output()
+
+ root_directory = self.get_virtual_directory()
+ scratch_directory = self._get_scratch_directory()
+
+ # TODO Check whether command exists in virtual root directory
+ # if not self._has_command(command[0], env):
+ # raise SandboxError("Staged artifacts do not provide command "
+ # "'{}'".format(command[0]),
+ # reason='missing-command')
+
+ # Grab the full path of the buildbox binary
+ buildbox_command = [utils.get_host_tool('buildbox')]
+
+ marked_directories = self._get_marked_directories()
+ for mark in marked_directories:
+ path = mark['directory']
+ assert path.startswith('/') and len(path) > 1
+ root_directory.descend(*path[1:].split('/'), create=True)
+
+ with open(os.path.join(scratch_directory, 'in'), 'wb') as input_digest_file:
+ digest = root_directory._get_digest()
+ input_digest_file.write(digest.SerializeToString())
+
+ buildbox_command += ["--local=" + root_directory.cas_cache.casdir]
+ buildbox_command += ["--input-digest=in"]
+ buildbox_command += ["--output-digest=out"]
+
+ if not flags & SandboxFlags.NETWORK_ENABLED:
+ # TODO
+ pass
+
+ if cwd is not None:
+ buildbox_command += ['--chdir=' + cwd]
+
+ # In interactive mode, we want a complete devpts inside
+ # the container, so there is a /dev/console and such. In
+ # the regular non-interactive sandbox, we want to hand pick
+ # a minimal set of devices to expose to the sandbox.
+ #
+ if flags & SandboxFlags.INTERACTIVE:
+ # TODO
+ pass
+
+ if flags & SandboxFlags.ROOT_READ_ONLY:
+ # TODO
+ pass
+
+ # Set UID and GID
+ if not flags & SandboxFlags.INHERIT_UID:
+ # TODO
+ pass
+
+ os.makedirs(os.path.join(scratch_directory, 'mnt'), exist_ok=True)
+ buildbox_command += ['mnt']
+
+ # Add the command
+ buildbox_command += command
+
+ # Use the MountMap context manager to ensure that any redirected
+ # mounts through fuse layers are in context and ready for buildbox
+ # to mount them from.
+ #
+ with ExitStack() as stack:
+ # Ensure the cwd exists
+ if cwd is not None and len(cwd) > 1:
+ assert cwd.startswith('/')
+ root_directory.descend(*cwd[1:].split('/'), create=True)
+
+ # If we're interactive, we want to inherit our stdin,
+ # otherwise redirect to /dev/null, ensuring process
+ # disconnected from terminal.
+ if flags & SandboxFlags.INTERACTIVE:
+ stdin = sys.stdin
+ else:
+ stdin = stack.enter_context(open(os.devnull, "r"))
+
+ # Run buildbox !
+ exit_code = self.run_buildbox(buildbox_command, stdin, stdout, stderr, env,
+ interactive=(flags & SandboxFlags.INTERACTIVE),
+ cwd=scratch_directory)
+
+ if exit_code == 0:
+ with open(os.path.join(scratch_directory, 'out'), 'rb') as output_digest_file:
+ output_digest = remote_execution_pb2.Digest()
+ output_digest.ParseFromString(output_digest_file.read())
+ self._vdir = CasBasedDirectory(root_directory.cas_cache, digest=output_digest)
+
+ return exit_code
+
+ def run_buildbox(self, argv, stdin, stdout, stderr, env, *, interactive, cwd):
+ def kill_proc():
+ if process:
+ # First attempt to gracefully terminate
+ proc = psutil.Process(process.pid)
+ proc.terminate()
+
+ try:
+ proc.wait(20)
+ except psutil.TimeoutExpired:
+ utils._kill_process_tree(process.pid)
+
+ def suspend_proc():
+ group_id = os.getpgid(process.pid)
+ os.killpg(group_id, signal.SIGSTOP)
+
+ def resume_proc():
+ group_id = os.getpgid(process.pid)
+ os.killpg(group_id, signal.SIGCONT)
+
+ with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
+ process = subprocess.Popen(
+ argv,
+ close_fds=True,
+ env=env,
+ stdin=stdin,
+ stdout=stdout,
+ stderr=stderr,
+ cwd=cwd,
+ start_new_session=interactive
+ )
+
+ # Wait for the child process to finish, ensuring that
+ # a SIGINT has exactly the effect the user probably
+ # expects (i.e. let the child process handle it).
+ try:
+ while True:
+ try:
+ _, status = os.waitpid(process.pid, 0)
+ # If the process exits due to a signal, we
+ # brutally murder it to avoid zombies
+ if not os.WIFEXITED(status):
+ utils._kill_process_tree(process.pid)
+
+ # Unlike in the bwrap case, here only the main
+ # process seems to receive the SIGINT. We pass
+ # on the signal to the child and then continue
+ # to wait.
+ except KeyboardInterrupt:
+ process.send_signal(signal.SIGINT)
+ continue
+
+ break
+ # If we can't find the process, it has already died of
+ # its own accord, and therefore we don't need to check
+ # or kill anything.
+ except psutil.NoSuchProcess:
+ pass
+
+ # Return the exit code - see the documentation for
+ # os.WEXITSTATUS to see why this is required.
+ if os.WIFEXITED(status):
+ exit_code = os.WEXITSTATUS(status)
+ else:
+ exit_code = -1
+
+ return exit_code
+
+ def _use_cas_based_directory(self):
+ # Always use CasBasedDirectory for BuildBox
+ return True