You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:03:00 UTC
[buildstream] branch
218-allow-specifying-the-chroot-binary-to-use-for-sandboxes-on-unix-platforms
created (now 85b5209)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch 218-allow-specifying-the-chroot-binary-to-use-for-sandboxes-on-unix-platforms
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 85b5209 Temp
This branch includes the following new commits:
new 85b5209 Temp
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: Temp
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch 218-allow-specifying-the-chroot-binary-to-use-for-sandboxes-on-unix-platforms
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 85b52097f981c665bfb80df7ddccadf43d3ec8d9
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Wed Jan 31 14:04:43 2018 +0000
Temp
---
buildstream/_platform/unix.py | 10 +-
buildstream/sandbox/__init__.py | 1 +
buildstream/sandbox/_sandboxchroot.py | 139 +++++++++++++-------------
buildstream/sandbox/_sandboxuserchroot.py | 158 ++++++++++++++++++++++++++++++
4 files changed, 238 insertions(+), 70 deletions(-)
diff --git a/buildstream/_platform/unix.py b/buildstream/_platform/unix.py
index 6d7b463..7c3a805 100644
--- a/buildstream/_platform/unix.py
+++ b/buildstream/_platform/unix.py
@@ -22,7 +22,7 @@ import os
from .._artifactcache.tarcache import TarCache
from .._exceptions import PlatformError
-from ..sandbox import SandboxChroot
+from ..sandbox import SandboxChroot, SandboxUserChroot
from . import Platform
@@ -35,12 +35,14 @@ class Unix(Platform):
self._artifact_cache = TarCache(context)
# Not necessarily 100% reliable, but we want to fail early.
- if os.geteuid() != 0:
- raise PlatformError("Root privileges are required to run without bubblewrap.")
+ # if os.geteuid() != 0:
+ # raise PlatformError("Root privileges are required to run without bubblewrap.")
@property
def artifactcache(self):
return self._artifact_cache
def create_sandbox(self, *args, **kwargs):
- return SandboxChroot(*args, **kwargs)
+ # We can optionally create a SandboxUserChroot
+ return SandboxUserChroot(*args, **kwargs)
+ # return SandboxChroot(*args, **kwargs)
diff --git a/buildstream/sandbox/__init__.py b/buildstream/sandbox/__init__.py
index 7ee871c..c55e53e 100644
--- a/buildstream/sandbox/__init__.py
+++ b/buildstream/sandbox/__init__.py
@@ -19,5 +19,6 @@
# Tristan Maat <tr...@codethink.co.uk>
from .sandbox import Sandbox, SandboxFlags
+from ._sandboxuserchroot import SandboxUserChroot
from ._sandboxchroot import SandboxChroot
from ._sandboxbwrap import SandboxBwrap
diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py
index 584f0e1..8ac0ba4 100644
--- a/buildstream/sandbox/_sandboxchroot.py
+++ b/buildstream/sandbox/_sandboxchroot.py
@@ -91,24 +91,12 @@ class SandboxChroot(Sandbox):
return status
- # chroot()
+ # popen()
#
- # A helper function to chroot into the rootfs.
+ # A helper function to create and manage a subprocess. We mimic
+ # subprocess.Popen's interface here.
#
- # Args:
- # rootfs (str): The path of the sysroot to chroot into
- # command (list): The command to execute in the chroot env
- # stdin (file): The stdin
- # stdout (file): The stdout
- # stderr (file): The stderr
- # cwd (str): The current working directory
- # env (dict): The environment variables to use while executing the command
- # flags (:class:`SandboxFlags`): The flags to enable on the sandbox
- #
- # Returns:
- # (int): The exit code of the executed command
- #
- def chroot(self, rootfs, command, stdin, stdout, stderr, cwd, env, flags):
+ def popen(self, command, **kwargs):
def kill_proc():
if process:
# First attempt to gracefully terminate
@@ -128,55 +116,76 @@ class SandboxChroot(Sandbox):
group_id = os.getpgid(process.pid)
os.killpg(group_id, signal.SIGCONT)
+ with _signals.suspendable(suspend_proc, resume_proc), \
+ _signals.terminator(kill_proc):
+ process = subprocess.Popen(command, **kwargs)
+
+ # Wait for the child process to finish, ensuring that
+ # a SIGINT has exactly the effect the user probably
+ # expects (i.e. let the child process handle it).
+ try:
+ while True:
+ try:
+ _, status = os.waitpid(process.pid, 0)
+ # If the process exits due to a signal, we
+ # brutally murder it to avoid zombies
+ if not os.WIFEXITED(status):
+ utils._kill_process_tree(process.pid)
+
+ # Unlike in the bwrap case, here only the main
+ # process seems to receive the SIGINT. We pass
+ # on the signal to the child and then continue
+ # to wait.
+ except KeyboardInterrupt:
+ process.send_signal(signal.SIGINT)
+ continue
+
+ break
+ # If we can't find the process, it has already died of
+ # its own accord, and therefore we don't need to check
+ # or kill anything.
+ except psutil.NoSuchProcess:
+ pass
+
+ # Return the exit code - see the documentation for
+ # os.WEXITSTATUS to see why this is required.
+ if os.WIFEXITED(status):
+ code = os.WEXITSTATUS(status)
+ else:
+ code = -1
+
+ return code
+
+ # chroot()
+ #
+ # A helper function to chroot into the rootfs.
+ #
+ # Args:
+ # rootfs (str): The path of the sysroot to chroot into
+ # command (list): The command to execute in the chroot env
+ # stdin (file): The stdin
+ # stdout (file): The stdout
+ # stderr (file): The stderr
+ # cwd (str): The current working directory
+ # env (dict): The environment variables to use while executing the command
+ # flags (:class:`SandboxFlags`): The flags to enable on the sandbox
+ #
+ # Returns:
+ # (int): The exit code of the executed command
+ #
+ def chroot(self, rootfs, command, stdin, stdout, stderr, cwd, env, flags):
try:
- with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
- process = subprocess.Popen(
- command,
- close_fds=True,
- cwd=os.path.join(rootfs, cwd.lstrip(os.sep)),
- env=env,
- stdin=stdin,
- stdout=stdout,
- stderr=stderr,
- # If you try to put gtk dialogs here Tristan (either)
- # will personally scald you
- preexec_fn=lambda: (os.chroot(rootfs), os.chdir(cwd)),
- start_new_session=flags & SandboxFlags.INTERACTIVE
- )
-
- # Wait for the child process to finish, ensuring that
- # a SIGINT has exactly the effect the user probably
- # expects (i.e. let the child process handle it).
- try:
- while True:
- try:
- _, status = os.waitpid(process.pid, 0)
- # If the process exits due to a signal, we
- # brutally murder it to avoid zombies
- if not os.WIFEXITED(status):
- utils._kill_process_tree(process.pid)
-
- # Unlike in the bwrap case, here only the main
- # process seems to receive the SIGINT. We pass
- # on the signal to the child and then continue
- # to wait.
- except KeyboardInterrupt:
- process.send_signal(signal.SIGINT)
- continue
-
- break
- # If we can't find the process, it has already died of
- # its own accord, and therefore we don't need to check
- # or kill anything.
- except psutil.NoSuchProcess:
- pass
-
- # Return the exit code - see the documentation for
- # os.WEXITSTATUS to see why this is required.
- if os.WIFEXITED(status):
- code = os.WEXITSTATUS(status)
- else:
- code = -1
+ return self.popen(command,
+ close_fds=True,
+ cwd=os.path.join(rootfs, cwd.lstrip(os.sep)),
+ env=env,
+ stdin=stdin,
+ stdout=stdout,
+ stderr=stderr,
+ # If you try to put gtk dialogs here Tristan (either)
+ # will personally scald you
+ preexec_fn=lambda: (os.chroot(rootfs), os.chdir(cwd)),
+ start_new_session=flags & SandboxFlags.INTERACTIVE)
except subprocess.SubprocessError as e:
# Exceptions in preexec_fn are simply reported as
@@ -189,8 +198,6 @@ class SandboxChroot(Sandbox):
else:
raise SandboxError('Could not run command {}: {}'.format(command, e)) from e
- return code
-
# create_devices()
#
# Create the nodes in /dev/ usually required for builds (null,
diff --git a/buildstream/sandbox/_sandboxuserchroot.py b/buildstream/sandbox/_sandboxuserchroot.py
new file mode 100644
index 0000000..6e98e8d
--- /dev/null
+++ b/buildstream/sandbox/_sandboxuserchroot.py
@@ -0,0 +1,158 @@
+import os
+import pwd
+import sys
+import stat
+import subprocess
+from pathlib import Path
+from contextlib import ExitStack, contextmanager
+
+from .. import utils
+from . import SandboxFlags
+from ._mount import MountMap
+from .._exceptions import SandboxError
+from ._sandboxchroot import SandboxChroot
+from .._message import Message, MessageType
+
+
+# The sandbox directory needs to fulfill a few criteria:
+# - Its parents must be owned by root
+# - It and its children must be owned by the user defined in the
+# configuration file
+# - Neither it nor its parents must be more permissive than 755
+# - It cannot be in a directory mounted into the sandbox (duh)
+#
+# If we allow the user to specify this location (we probably should),
+# those criteria would be nice to check for before sandbox execution.
+# Although userchroot itself checks for some, the error messages are
+# not particularly helpful.
+#
+SANDBOX_DIR = '/usr/local/sandboxes'
+
+
+def assert_userchroot_configuration():
+ configured = False
+ user = pwd.getpwuid(os.getuid())[0]
+ userchroot = utils.get_host_tool('userchroot')
+ config = Path(userchroot).parents[1].joinpath('etc/userchroot.conf')
+
+ if config.exists():
+ with open(config, 'r') as configf:
+ for line in configf:
+ if line.rstrip() == '{}:{}'.format(user, SANDBOX_DIR):
+ configured = True
+ break
+
+ if not configured:
+ raise SandboxError("'userchroot' is not configured correctly. "
+ "Please add '{}:{}' to '{}'"
+ .format(user, SANDBOX_DIR, config))
+
+
+class SandboxUserChroot(SandboxChroot):
+ def run(self, command, flags, *, cwd=None, env=None):
+ # Ensure sandbox default configuration
+ if cwd is None:
+ cwd = self._get_work_directory() or '/'
+
+ if env is None:
+ env = self._get_environment()
+
+ if isinstance(command, str):
+ command = [command]
+
+ stdout, stderr = self._get_output()
+
+ # Create the mount map, this will tell us where
+ # each mount point needs to be mounted from and to
+ self.mount_map = MountMap(self, True)
+
+ # Make sure userchroot is configured correctly
+ assert_userchroot_configuration()
+
+ with ExitStack() as stack:
+ # Create sysroot
+ try:
+ os.makedirs(SANDBOX_DIR, exist_ok=True)
+ rootfs = stack.enter_context(utils._tempdir(dir=SANDBOX_DIR))
+ except PermissionError as e:
+ raise SandboxError('Could not create sysroot in {}: {}'
+ .format(SANDBOX_DIR, e)) from e
+
+ stack.enter_context(self.stage_sysroot(rootfs, flags, stdout, stderr))
+
+ # Chroot!
+ if flags & SandboxFlags.INTERACTIVE:
+ stdin = sys.stdin
+ else:
+ stdin = stack.enter_context(open(os.devnull, 'r'))
+
+ status = self.chroot(rootfs, command, stdin, stdout, stderr,
+ cwd, env, flags)
+ return status
+
+ def chroot(self, rootfs, command, stdin, stdout, stderr, cwd, env,
+ flags):
+ # Create a script in the root directory of the sysroot to
+ # execute the given commands.
+ script = "\n".join(["#!/bin/sh"] + command)
+ scriptpath = os.path.join(rootfs, 'buildstream-run.sh')
+
+ with open(scriptpath, 'w') as scriptfile:
+ scriptfile.write(script)
+ perms = os.stat(scriptpath).st_mode
+ os.chmod(scriptpath, perms & stat.S_IXUSR)
+
+ # Execute the script with userchroot
+ try:
+ command = [utils.get_host_tool('userchroot'),
+ rootfs,
+ '--install-devices',
+ '/buildstream-run.sh']
+ return self.popen(command,
+ env=env,
+ stdin=stdin,
+ stdout=stdout,
+ stderr=stderr,
+ cwd=os.path.join(rootfs, cwd.lstrip(os.sep)),
+ start_new_session=flags & SandboxFlags.INTERACTIVE)
+
+ except subprocess.SubprocessError as e:
+ raise SandboxError('Could not run command {}: {}'.format(command, e)) from e
+
+ # mount_dirs()
+ #
+ # Since we aren't root we can't arbitrarily mount directories. Yet
+ # we *require* our FUSE filesystem for at least some operations.
+ #
+ # FUSE can be mounted by users, therefore this mount function
+ # attempts to safely mount our FUSE system on top of a copy of our
+ # sandbox files.
+ #
+ # This *does* mean that this platform is significantly slower than
+ # others, unfortunately...
+ #
+ @contextmanager
+ def stage_sysroot(self, rootfs, flags, stdout, stderr):
+ def mount(d):
+ overrides = self._get_mount_sources()
+
+ if d in overrides:
+ src = overrides[d]
+ else:
+ src = self.mount_map.get_mount_source(d)
+
+ dst = os.path.join(rootfs, d.lstrip(os.sep))
+
+ self.info('Mounting {} to {}'.format(src, dst))
+
+ with self.mount_map.mounted(rootfs):
+ yield
+
+ mount('/')
+
+ for mark in self._get_marked_directories():
+ mount(mark['directory'])
+
+ def info(self, message):
+ msg = Message('sandbox', MessageType.INFO, message)
+ self._get_context()._message(msg)