You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:46:12 UTC
[buildstream] branch tlater/casd-socket-permissions created (now
0a8126c)
This is an automated email from the ASF dual-hosted git repository.
root pushed a change to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 0a8126c tar.py: Make directories with incorrect permissions traversable
This branch includes the following new commits:
new abaa1d1 casserver.py: Add logging
new 7c6d27b casserver.py: Run buildbox-casd in `bst-artifact-server`
new e8ddfe1 casserver.py: Move CASCache API to a smaller, local class
new 2a02568 casserver.py: Proxy CAS requests to buildbox-casd
new fdc253a casserver.py: Make BuildStream util imports explicit
new c424e99 Remove newly unused API surfaces in CASCache
new 502ea5c casserver.py: Use FetchTree instead of directly updating mtimes
new 44bd2d7 WIP: Temporarily disable the ability to link in CasBasedDirectory
new 110cc5d WIP: Temporarily disable source determinism tests
new 5fc3835 Make temporary staging directories group-accessible
new cdbbe11 workspaces.py: Make workspace config group-readable
new 5822e0f element.py: Make build directory a proper utils._tempdir
new 288bd39 Make roundtrip_dump allow group permissions
new bcad8f4 cascache.py: Set up socket path via a symlink
new cf42e87 WIP: casserver.py: Adapt make_socket_path()
new 397ddcf plugins/sources/tar.py: Ensure read permissions for groups as well
new 2c93f0c testing/integration.py: Make the integration cache group-readable
new f5854ee Make junction test subproject configuration group-readable
new 0a8126c tar.py: Make directories with incorrect permissions traversable
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/19: casserver.py: Run buildbox-casd in
`bst-artifact-server`
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7c6d27b5dcfd4434a759febeae9195995f44a304
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Oct 15 11:38:29 2019 +0100
casserver.py: Run buildbox-casd in `bst-artifact-server`
This is in preparation of a switch to directly calling buildbox-casd
through the `bst-artifact-server`, rather than relying on BuildStream
internals to do this.
This should help untangle the codebase a little, since our CAS
interfaces will all end up with a single goal, rather than trying to
do both server-end and client-end things.
---
src/buildstream/_cas/casserver.py | 121 +++++++++++++++++++++++++++++++++++++-
1 file changed, 120 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index d5a29a3..c0c62b0 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -18,13 +18,15 @@
# Jürg Billeter <ju...@codethink.co.uk>
from concurrent import futures
-from contextlib import contextmanager
from enum import Enum
+import contextlib
import logging
import os
import signal
+import subprocess
import sys
import tempfile
+import time
import uuid
import errno
@@ -78,6 +80,120 @@ class ClickLogLevel(click.Choice):
return LogLevel(super().convert(value, param, ctx))
+# CASdRunner():
+#
+# Manage a buildbox-casd process.
+#
+# FIXME: Probably better to replace this with the work from !1638
+#
+class CASdRunner:
+ def __init__(self, path: str, *, cache_quota: int = None, log_level: LogLevel = LogLevel.WARNING):
+ self.root = path
+ self.casdir = os.path.join(path, "cas")
+ self.tmpdir = os.path.join(path, "tmp")
+
+ self._casd_process = None
+ self._casd_socket_path = None
+ self._casd_socket_tempdir = None
+ self._log_level = log_level
+ self._quota = cache_quota
+
+ # start_casd():
+ #
+ # Start the CASd process.
+ #
+ def start_casd(self):
+ assert not self._casd_process, "CASd was already started"
+
+ os.makedirs(os.path.join(self.casdir, "refs", "heads"), exist_ok=True)
+ os.makedirs(os.path.join(self.casdir, "objects"), exist_ok=True)
+ os.makedirs(self.tmpdir, exist_ok=True)
+
+ # Place socket in global/user temporary directory to avoid hitting
+ # the socket path length limit.
+ self._casd_socket_tempdir = tempfile.mkdtemp(prefix="buildstream")
+ self._casd_socket_path = os.path.join(self._casd_socket_tempdir, "casd.sock")
+
+ casd_args = [utils.get_host_tool("buildbox-casd")]
+ casd_args.append("--bind=unix:" + self._casd_socket_path)
+ casd_args.append("--log-level=" + self._log_level.value)
+
+ if self._quota is not None:
+ casd_args.append("--quota-high={}".format(int(self._quota)))
+ casd_args.append("--quota-low={}".format(int(self._quota / 2)))
+
+ casd_args.append(self.root)
+
+ blocked_signals = signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGINT])
+
+ try:
+ self._casd_process = subprocess.Popen(
+ casd_args,
+ cwd=self.root,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+ finally:
+ signal.pthread_sigmask(signal.SIG_SETMASK, blocked_signals)
+
+ # stop():
+ #
+ # Stop and tear down the CASd process.
+ #
+ def stop(self):
+ return_code = self._casd_process.poll()
+
+ if return_code is not None:
+ self._casd_process = None
+ logging.error(
+ "Buildbox-casd died during the run. Exit code: %s", return_code
+ )
+ logging.error(self._casd_process.stdout.read().decode())
+ return
+
+ self._casd_process.terminate()
+
+ try:
+ return_code = self._casd_process.wait(timeout=0.5)
+ except subprocess.TimeoutExpired:
+ with contextlib.suppress():
+ try:
+ return_code = self._casd_process.wait(timeout=15)
+ except subprocess.TimeoutExpired:
+ self._casd_process.kill()
+ self._casd_process.wait(timeout=15)
+ logging.warning(
+ "Buildbox-casd didn't exit in time and has been killed"
+ )
+ logging.error(self._casd_process.stdout.read().decode())
+ self._casd_process = None
+ return
+
+ if return_code != 0:
+ logging.error(
+ "Buildbox-casd didn't exit cleanly. Exit code: %d", return_code
+ )
+ logging.error(self._casd_process.stdout.read().decode())
+
+ self._casd_process = None
+
+ # get_socket_path():
+ #
+ # Get the path to the socket of the CASd process - None if the
+ # process has not been started yet.
+ #
+ def get_socket_path(self) -> str:
+ assert self._casd_socket_path is not None, "CASd has not been started"
+ return self._casd_socket_path
+
+ # get_casdir():
+ #
+ # Get the path to the directory managed by CASd.
+ #
+ def get_casdir(self) -> str:
+ return self.casdir
+
+
# create_server():
#
# Create gRPC CAS artifact server as specified in the Remote Execution API.
@@ -96,6 +212,8 @@ def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.WA
logger.addHandler(handler)
cas = CASCache(os.path.abspath(repo), cache_quota=quota, protect_session_blobs=False)
+ cas_runner = CASdRunner(os.path.abspath(repo), cache_quota=quota)
+ cas_runner.start_casd()
try:
artifactdir = os.path.join(os.path.abspath(repo), 'artifacts', 'refs')
@@ -137,6 +255,7 @@ def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.WA
finally:
cas.release_resources()
+ cas_runner.stop()
@click.command(short_help="CAS Artifact Server")
[buildstream] 05/19: casserver.py: Make BuildStream util imports
explicit
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit fdc253a51559ed1e87fdef278d81df99aed3997b
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Wed Oct 16 13:06:23 2019 +0100
casserver.py: Make BuildStream util imports explicit
We'd rather not import too much from BuildStream, so making this
explicit saves some figuring out what we'd need to implement locally.
---
src/buildstream/_cas/casserver.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index d1bef68..e0063d1 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -40,7 +40,7 @@ from .._protos.google.bytestream import bytestream_pb2_grpc
from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
artifact_pb2, artifact_pb2_grpc, source_pb2, source_pb2_grpc
-from .. import utils
+from ..utils import save_file_atomic, get_host_tool
# The default limit for gRPC messages is 4 MiB.
@@ -200,7 +200,7 @@ class CASCache:
ref_path = self.ref_path(ref)
os.makedirs(os.path.dirname(ref_path), exist_ok=True)
- with utils.save_file_atomic(ref_path, 'wb', tempdir=self.tmpdir) as f:
+ with save_file_atomic(ref_path, 'wb', tempdir=self.tmpdir) as f:
f.write(tree.SerializeToString())
# resolve_ref():
@@ -337,7 +337,7 @@ class CASdRunner:
self._casd_socket_tempdir = tempfile.mkdtemp(prefix="buildstream")
self._casd_socket_path = os.path.join(self._casd_socket_tempdir, "casd.sock")
- casd_args = [utils.get_host_tool("buildbox-casd")]
+ casd_args = [get_host_tool("buildbox-casd")]
casd_args.append("--bind=unix:" + self._casd_socket_path)
casd_args.append("--log-level=" + self._log_level.value)
@@ -741,7 +741,7 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
# Add the artifact proto to the cas
artifact_path = os.path.join(self.artifactdir, request.cache_key)
os.makedirs(os.path.dirname(artifact_path), exist_ok=True)
- with utils.save_file_atomic(artifact_path, mode='wb') as f:
+ with save_file_atomic(artifact_path, mode='wb') as f:
f.write(artifact.SerializeToString())
return artifact
@@ -818,7 +818,7 @@ class _SourceServicer(source_pb2_grpc.SourceServiceServicer):
def _set_source(self, cache_key, source_proto):
path = os.path.join(self.sourcedir, cache_key)
os.makedirs(os.path.dirname(path), exist_ok=True)
- with utils.save_file_atomic(path, 'w+b') as f:
+ with save_file_atomic(path, 'w+b') as f:
f.write(source_proto.SerializeToString())
[buildstream] 10/19: Make temporary staging directories
group-accessible
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5fc38355ae19aef73ffe024171ab7028a4dfefd2
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Nov 5 11:32:38 2019 +0000
Make temporary staging directories group-accessible
This is again required because of the userchroot workflow requiring
two UIDs. By default, python will create temporary directories with
u+rwx,og-rwx, so that we don't leak information to the public /tmp.
This is irrelevant here since we are in directories that shouldn't be
as easily accessible anyway (since they are usually far inside
"$HOME", or at least configured by the sysadmin to be reasonably
safe), and a hindrance to adopt userchroot as a first-class sandbox.
---
src/buildstream/_artifact.py | 4 ++++
src/buildstream/_sourcecache.py | 2 +-
src/buildstream/element.py | 2 +-
src/buildstream/utils.py | 43 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 49 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index e5174ea..711d402 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -163,7 +163,11 @@ class Artifact():
# Store public data
with utils._tempnamedfile_name(dir=self._tmpdir) as tmpname:
+ # FIXME: This overrides the original file. Should check if
+ # that has side-effects besides re-setting the permissions
+ # (hence _group_tempnamedfile_name is useless here).
_yaml.roundtrip_dump(publicdata, tmpname)
+ os.chmod(tmpname, utils.URWX_GRWX)
public_data_digest = self._cas.add_object(path=tmpname, link_directly=True)
artifact.public_data.CopyFrom(public_data_digest)
size += public_data_digest.size_bytes
diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py
index cdbe5b9..7708f3d 100644
--- a/src/buildstream/_sourcecache.py
+++ b/src/buildstream/_sourcecache.py
@@ -185,7 +185,7 @@ class SourceCache(BaseCache):
vdir.import_files(self.export(previous_source))
if not source.BST_STAGE_VIRTUAL_DIRECTORY:
- with utils._tempdir(dir=self.context.tmpdir, prefix='staging-temp') as tmpdir:
+ with utils._group_tempdir(dir=self.context.tmpdir, prefix='staging-temp') as tmpdir:
if not vdir.is_empty():
vdir.export_files(tmpdir)
source._stage(tmpdir)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 5fa8f14..7bb8bc0 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1453,7 +1453,7 @@ class Element(Plugin):
# It's advantageous to have this temporary directory on
# the same file system as the rest of our cache.
with self.timed_activity("Staging sources", silent_nested=True), \
- utils._tempdir(dir=context.tmpdir, prefix='staging-temp') as temp_staging_directory:
+ utils._group_tempdir(dir=context.tmpdir, prefix='staging-temp') as temp_staging_directory:
import_dir = temp_staging_directory
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index de7c14b..e9f0fb7 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -65,6 +65,16 @@ _INITIAL_NUM_THREADS_IN_MAIN_PROCESS = 1
# Number of seconds to wait for background threads to exit.
_AWAIT_THREADS_TIMEOUT_SECONDS = 5
+# Bit flags for ug+rwx,o-rwx permissions
+URWX_GRWX = (
+ stat.S_IRUSR |
+ stat.S_IWUSR |
+ stat.S_IXUSR |
+ stat.S_IRGRP |
+ stat.S_IWGRP |
+ stat.S_IXGRP
+)
+
class UtilError(BstError):
"""Raised by utility functions when system calls fail.
@@ -1010,6 +1020,32 @@ def _tempdir(suffix="", prefix="tmp", dir=None): # pylint: disable=redefined-bu
cleanup_tempdir()
+# _group_tempdir()
+#
+# Same as _tempdir(), but it allows RWX access to the entire user
+# *group*, instead of just the user.
+#
+# NOTE: This is potentially insecure. If created in a directory with
+# too open permissions, this will allow all users of the same
+# group to read files in here, which may leak
+# information. *Only* use this in directories whose parents are
+# more tightly controlled (i.e., non-public directories).
+#
+# Args:
+# dir (str): A path to a parent directory for the temporary directory
+# suffix (str): A suffix for the temproary directory name
+# prefix (str): A prefix for the temporary directory name
+#
+# Yields:
+# (str): The temporary directory
+#
+@contextmanager
+def _group_tempdir(**kwargs):
+ with _tempdir() as tempdir:
+ os.chmod(tempdir, URWX_GRWX)
+ yield tempdir
+
+
# _tempnamedfile()
#
# A context manager for doing work on an open temporary file
@@ -1100,6 +1136,13 @@ def _tempnamedfile_name(dir): # pylint: disable=redefined-builtin
rm_tempfile()
+@contextmanager
+def _group_tempnamedfile_name(dir):
+ with _tempnamedfile_name(dir) as temp:
+ os.chmod(temp, URWX_GRWX)
+ yield temp
+
+
# _kill_process_tree()
#
# Brutally murder a process and all of its children
[buildstream] 03/19: casserver.py: Move CASCache API to a smaller,
local class
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e8ddfe1d755f0093305eb3b98d630db9ffaeca81
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Oct 15 17:42:07 2019 +0100
casserver.py: Move CASCache API to a smaller, local class
---
src/buildstream/_cas/casserver.py | 225 +++++++++++++++++++++++++++++++++-----
1 file changed, 198 insertions(+), 27 deletions(-)
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index c0c62b0..4f07639 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -19,6 +19,7 @@
from concurrent import futures
from enum import Enum
+from typing import Set
import contextlib
import logging
import os
@@ -41,15 +42,178 @@ from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
artifact_pb2, artifact_pb2_grpc, source_pb2, source_pb2_grpc
from .. import utils
-from .._exceptions import CASError, CASCacheError
-from .cascache import CASCache
# The default limit for gRPC messages is 4 MiB.
# Limit payload to 1 MiB to leave sufficient headroom for metadata.
_MAX_PAYLOAD_BYTES = 1024 * 1024
+# CASCache:
+#
+# A slimmed down version of `buildstream._cas.cascache.CASCache` -
+# exposes exactly the bits of interface we need to update objects on
+# access.
+#
+# Note: This class *is* somewhat specialized and doesn't exactly do
+# what `buildstream._cas.cascache.CASCache` does anymore.
+#
+# Ideally this should be supported by buildbox-casd in the future.
+#
+class CASCache:
+ def __init__(self, root: str):
+ self.root = root
+ self.casdir = os.path.join(root, "cas")
+ self.tmpdir = os.path.join(root, "tmp")
+
+ # ref_path():
+ #
+ # Get the path to a digest's file.
+ #
+ # Args:
+ # ref - The ref of the digest.
+ #
+ # Returns:
+ # str - The path to the digest's file.
+ #
+ def ref_path(self, ref: str) -> str:
+ return os.path.join(self.casdir, 'refs', 'heads', ref)
+
+ # object_path():
+ #
+ # Get the path to an object's file.
+ #
+ # Args:
+ # digest - The digest of the object.
+ #
+ # Returns:
+ # str - The path to the object's file.
+ #
+ def object_path(self, digest) -> str:
+ return os.path.join(self.casdir, 'objects', digest.hash[:2], digest.hash[2:])
+
+ # remove_ref():
+ #
+ # Remove a digest file.
+ #
+ # Args:
+ # ref - The ref of the digest.
+ #
+ # Raises:
+ # FileNotFoundError - If the ref doesn't exist.
+ def remove_ref(self, ref: str):
+ basedir = os.path.join(self.casdir, 'refs', 'heads')
+
+ os.unlink(self.ref_path(ref))
+
+ # Now remove any leading directories
+ components = list(os.path.split(ref))
+ while components:
+ components.pop()
+ refdir = os.path.join(basedir, *components)
+
+ # Break out once we reach the base
+ if refdir == basedir:
+ break
+
+ try:
+ os.rmdir(refdir)
+ except FileNotFoundError:
+ # The parent directory did not exist, but it's
+ # parent directory might still be ready to prune
+ pass
+ except OSError as e:
+ if e.errno == errno.ENOTEMPTY:
+ # The parent directory was not empty, so we
+ # cannot prune directories beyond this point
+ break
+ raise
+
+ # set_ref():
+ #
+ # Create or update a ref with a new digest.
+ #
+ # Args:
+ # ref - The ref of the digest.
+ # tree - The digest to write.
+ #
+ def set_ref(self, ref: str, tree):
+ ref_path = self.ref_path(ref)
+
+ os.makedirs(os.path.dirname(ref_path), exist_ok=True)
+ with utils.save_file_atomic(ref_path, 'wb', tempdir=self.tmpdir) as f:
+ f.write(tree.SerializeToString())
+
+ # resolve_ref():
+ #
+ # Read a digest given its ref.
+ #
+ # Args:
+ # ref - The ref of the digest.
+ #
+ # Returns:
+ # remote_execution-pb2.Digest - The digest.
+ #
+ # Raises:
+ # FileNotFoundError - If the ref doesn't exist.
+ def resolve_ref(self, ref: str):
+ digest = remote_execution_pb2.Digest()
+ with open(self.ref_path(ref), 'rb') as f:
+ digest.ParseFromString(f.read())
+ return digest
+
+ # resolve_digest():
+ #
+ # Read the directory corresponding to a digest.
+ #
+ # Args:
+ # digest - The digest corresponding to a directory.
+ #
+ # Returns:
+ # remote_execution_pb2.Directory - The directory.
+ #
+ # Raises:
+ # FileNotFoundError - If the digest object doesn't exist.
+ def resolve_digest(self, digest):
+ directory = remote_execution_pb2.Directory()
+ with open(self.object_path(digest), 'rb') as f:
+ directory.ParseFromString(f.read())
+ return directory
+
+ # update_tree_mtime():
+ #
+ # Update the mtimes of all files in a tree.
+ #
+ # Args:
+ # tree - The digest of the tree to update.
+ #
+ # Raises:
+ # FileNotFoundError - If any of the tree's objects don't exist.
+ def update_tree_mtime(self, tree):
+ visited = set() # type: Set[str]
+ os.utime(self.object_path(tree))
+
+ def update_directory_node(node):
+ try:
+ if node.hash in visited:
+ return
+ except AttributeError:
+ raise Exception(type(node))
+
+ os.utime(self.object_path(node))
+ visited.add(node.hash)
+
+ directory = self.resolve_digest(node)
+ for filenode in directory.files: # pylint: disable=no-member
+ os.utime(self.object_path(filenode.digest))
+ for dirnode in directory.directories: # pylint: disable=no-member
+ update_directory_node(dirnode.digest)
+
+ # directory = self.resolve_digest(tree)
+ # update_directory_node(directory)
+ update_directory_node(tree)
+
+
# LogLevel():
#
# Represents the buildbox-casd log level.
@@ -211,13 +375,13 @@ def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.WA
handler.setLevel(LogLevel.get_logging_equivalent(log_level))
logger.addHandler(handler)
- cas = CASCache(os.path.abspath(repo), cache_quota=quota, protect_session_blobs=False)
cas_runner = CASdRunner(os.path.abspath(repo), cache_quota=quota)
cas_runner.start_casd()
+ cas_cache = CASCache(os.path.abspath(repo))
try:
- artifactdir = os.path.join(os.path.abspath(repo), 'artifacts', 'refs')
- sourcedir = os.path.join(os.path.abspath(repo), 'source_protos')
+ root = os.path.abspath(repo)
+ sourcedir = os.path.join(root, 'source_protos')
# Use max_workers default from Python 3.5+
max_workers = (os.cpu_count() or 1) * 5
@@ -234,10 +398,10 @@ def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.WA
_CapabilitiesServicer(), server)
buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(
- _ReferenceStorageServicer(cas, enable_push=enable_push), server)
+ _ReferenceStorageServicer(cas, cas_cache, enable_push=enable_push), server)
artifact_pb2_grpc.add_ArtifactServiceServicer_to_server(
- _ArtifactServicer(cas, artifactdir, update_cas=not index_only), server)
+ _ArtifactServicer(cas, root, cas_cache, update_cas=not index_only), server)
source_pb2_grpc.add_SourceServiceServicer_to_server(
_SourceServicer(sourcedir), server)
@@ -564,9 +728,10 @@ class _CapabilitiesServicer(remote_execution_pb2_grpc.CapabilitiesServicer):
class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
- def __init__(self, cas, *, enable_push):
+ def __init__(self, cas, cas_cache, *, enable_push):
super().__init__()
self.cas = cas
+ self.cas_cache = cas_cache
self.enable_push = enable_push
self.logger = logging.getLogger("casserver")
@@ -575,17 +740,17 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
response = buildstream_pb2.GetReferenceResponse()
try:
- tree = self.cas.resolve_ref(request.key, update_mtime=True)
+ tree = self.cas_cache.resolve_ref(request.key)
try:
- self.cas.update_tree_mtime(tree)
+ self.cas_cache.update_tree_mtime(tree)
except FileNotFoundError:
- self.cas.remove(request.key)
+ self.cas_cache.remove_ref(request.key)
context.set_code(grpc.StatusCode.NOT_FOUND)
return response
response.digest.hash = tree.hash
response.digest.size_bytes = tree.size_bytes
- except CASError:
+ except FileNotFoundError:
context.set_code(grpc.StatusCode.NOT_FOUND)
return response
@@ -599,7 +764,7 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
return response
for key in request.keys:
- self.cas.set_ref(key, request.digest)
+ self.cas_cache.set_ref(key, request.digest)
return response
@@ -614,10 +779,11 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
- def __init__(self, cas, artifactdir, *, update_cas=True):
+ def __init__(self, cas, root, cas_cache, *, update_cas=True):
super().__init__()
self.cas = cas
- self.artifactdir = artifactdir
+ self.cas_cache = cas_cache
+ self.artifactdir = os.path.join(root, 'artifacts', 'refs')
self.update_cas = update_cas
os.makedirs(artifactdir, exist_ok=True)
self.logger = logging.getLogger("casserver")
@@ -651,20 +817,20 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
try:
if str(artifact.files):
- self.cas.update_tree_mtime(artifact.files)
+ self.cas_cache.update_tree_mtime(artifact.files)
if str(artifact.buildtree):
# buildtrees might not be there
try:
- self.cas.update_tree_mtime(artifact.buildtree)
+ self.cas_cache.update_tree_mtime(artifact.buildtree)
except FileNotFoundError:
pass
if str(artifact.public_data):
- os.utime(self.cas.objpath(artifact.public_data))
+ os.utime(self.cas_cache.object_path(artifact.public_data))
for log_file in artifact.logs:
- os.utime(self.cas.objpath(log_file.digest))
+ os.utime(self.cas_cache.object_path(log_file.digest))
except FileNotFoundError:
os.unlink(artifact_path)
@@ -708,20 +874,25 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
def _check_directory(self, name, digest, context):
try:
- directory = remote_execution_pb2.Directory()
- with open(self.cas.objpath(digest), 'rb') as f:
- directory.ParseFromString(f.read())
+ self.cas_cache.resolve_digest(digest)
except FileNotFoundError:
- self.logger.warning("Artifact %s specified but no files found (%s)", name, self.cas.objpath(digest))
- context.abort(grpc.StatusCode.FAILED_PRECONDITION,
- "Artifact {} specified but no files found".format(name))
+ self.logger.warning(
+ "Artifact %s specified but no files found (%s)",
+ name,
+ self.cas_cache.object_path(digest))
+ context.abort(
+ grpc.StatusCode.FAILED_PRECONDITION,
+ "Artifact {} specified but no files found".format(name))
except DecodeError:
- self.logger.warning("Artifact %s specified but directory not found (%s)", name, self.cas.objpath(digest))
+ self.logger.warning(
+ "Artifact %s specified but directory not found (%s)",
+ name,
+ self.cas_cache.object_path(digest))
context.abort(grpc.StatusCode.FAILED_PRECONDITION,
"Artifact {} specified but directory not found".format(name))
def _check_file(self, name, digest, context):
- if not os.path.exists(self.cas.objpath(digest)):
+ if not os.path.exists(self.cas_cache.object_path(digest)):
context.abort(grpc.StatusCode.FAILED_PRECONDITION,
"Artifact {} specified but not found".format(name))
[buildstream] 17/19: testing/integration.py: Make the integration
cache group-readable
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2c93f0cc6e0962baf88c3c89059987988deaba03
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Fri Nov 8 12:25:46 2019 +0000
testing/integration.py: Make the integration cache group-readable
---
src/buildstream/testing/integration.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/src/buildstream/testing/integration.py b/src/buildstream/testing/integration.py
index 01635de..c329abe 100644
--- a/src/buildstream/testing/integration.py
+++ b/src/buildstream/testing/integration.py
@@ -24,6 +24,7 @@ integration tests.
import os
import shutil
+import stat
import tempfile
import pytest
@@ -68,6 +69,15 @@ class IntegrationCache:
# the artifacts directory
try:
self.cachedir = tempfile.mkdtemp(dir=self.root, prefix='cache-')
+ os.chmod(
+ self.cachedir,
+ stat.S_IRUSR |
+ stat.S_IWUSR |
+ stat.S_IXUSR |
+ stat.S_IRGRP |
+ stat.S_IWGRP |
+ stat.S_IXGRP
+ )
except OSError as e:
raise AssertionError("Unable to create test directory !") from e
[buildstream] 15/19: WIP: casserver.py: Adapt make_socket_path()
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cf42e87fa3980bedd49b60a0f64b422e16f861cc
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Mon Nov 11 15:17:00 2019 +0000
WIP: casserver.py: Adapt make_socket_path()
While this is necessary, it really just shows the need for a generic
CASDProcessManager.
---
src/buildstream/_cas/casserver.py | 63 +++++++++++++++++++++++++++++++++++++--
1 file changed, 61 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index 5af610f..bc308d0 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -30,6 +30,8 @@ import tempfile
import time
import uuid
import errno
+import random
+import stat
import grpc
from google.protobuf.message import DecodeError
@@ -350,8 +352,7 @@ class CASdRunner:
# Place socket in global/user temporary directory to avoid hitting
# the socket path length limit.
- self._casd_socket_tempdir = tempfile.mkdtemp(prefix="buildstream")
- self._casd_socket_path = os.path.join(self._casd_socket_tempdir, "casd.sock")
+ self._casd_socket_path = self._make_socket_path(self.root)
casd_args = [get_host_tool("buildbox-casd")]
casd_args.append("--bind=unix:" + self._casd_socket_path)
@@ -375,6 +376,64 @@ class CASdRunner:
finally:
signal.pthread_sigmask(signal.SIG_SETMASK, blocked_signals)
+ # _make_socket_path()
+ #
+ # Create a path to the CASD socket, ensuring that we don't exceed
+ # the socket path limit.
+ #
+ # Note that we *may* exceed the path limit if the python-chosen
+ # tmpdir path is very long, though this should be /tmp.
+ #
+ # Args:
+ # path (str): The root directory for the CAS repository.
+ #
+ # Returns:
+ # (str) - The path to the CASD socket.
+ #
+ def _make_socket_path(self, path):
+ self._casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream')
+ # mkdtemp will create this directory in the "most secure"
+ # way. This translates to "u+rwx,go-rwx".
+ #
+ # This is a good thing, generally, since it prevents us
+ # from leaking sensitive information to other users, but
+ # it's a problem for the workflow for userchroot, since
+ # the setuid casd binary will not share a uid with the
+ # user creating the tempdir.
+ #
+ # Instead, we chmod the directory 750, and only place a
+ # symlink to the CAS directory in here, which will allow the
+ # CASD process RWX access to a directory without leaking build
+ # information.
+ os.chmod(
+ self._casd_socket_tempdir,
+ stat.S_IRUSR |
+ stat.S_IWUSR |
+ stat.S_IXUSR |
+ stat.S_IRGRP |
+ stat.S_IXGRP,
+ )
+
+ os.symlink(path, os.path.join(self._casd_socket_tempdir, "cas"))
+ # FIXME: There is a potential race condition here; if multiple
+ # instances happen to create the same socket path, at least
+ # one will try to talk to the same server as us.
+ #
+ # There's no real way to avoid this from our side; we'd need
+ # buildbox-casd to tell us that it could not create a fresh
+ # socket.
+ #
+ # We could probably make this even safer by including some
+ # thread/process-specific information, but we're not really
+ # supporting this use case anyway; it's mostly here fore
+ # testing, and to help more gracefully handle the situation.
+ #
+ # Note: this uses the same random string generation principle
+ # as cpython, so this is probably a safe file name.
+ socket_name = "casserver-{}.sock".format(
+ "".join(random.choices("abcdefghijklmnopqrstuvwxyz0123456789_", k=8)))
+ return os.path.join(self._casd_socket_tempdir, "cas", socket_name)
+
# stop():
#
# Stop and tear down the CASd process.
[buildstream] 07/19: casserver.py: Use FetchTree instead of
directly updating mtimes
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 502ea5c578401794a93b8898074bbb1747c6e0ae
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Oct 29 17:01:53 2019 +0000
casserver.py: Use FetchTree instead of directly updating mtimes
---
src/buildstream/_cas/casserver.py | 92 +++++++++++++++++++++++++++------------
1 file changed, 63 insertions(+), 29 deletions(-)
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index e0063d1..5af610f 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -35,11 +35,20 @@ import grpc
from google.protobuf.message import DecodeError
import click
-from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
+from .._protos.build.bazel.remote.execution.v2 import (
+ remote_execution_pb2,
+ remote_execution_pb2_grpc,
+)
from .._protos.google.bytestream import bytestream_pb2_grpc
-from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
- artifact_pb2, artifact_pb2_grpc, source_pb2, source_pb2_grpc
-
+from .._protos.build.buildgrid import local_cas_pb2, local_cas_pb2_grpc
+from .._protos.buildstream.v2 import (
+ buildstream_pb2,
+ buildstream_pb2_grpc,
+ artifact_pb2,
+ artifact_pb2_grpc,
+ source_pb2,
+ source_pb2_grpc,
+)
from ..utils import save_file_atomic, get_host_tool
@@ -57,6 +66,7 @@ class CASRemote:
def __init__(self, url: str):
self._url = url
+ self._local_cas = None
self._bytestream = None
self._cas = None
@@ -94,6 +104,7 @@ class CASRemote:
raise
# Set up the RPC stubs
+ self._local_cas = local_cas_pb2_grpc.LocalContentAddressableStorageStub(self._channel)
self._bytestream = bytestream_pb2_grpc.ByteStreamStub(self._channel)
self._cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self._channel)
@@ -102,6 +113,11 @@ class CASRemote:
assert self._cas is not None, "CAS stub was not initialized"
return self._cas
+ def get_local_cas(self) -> local_cas_pb2_grpc.LocalContentAddressableStorageStub:
+ self._initialize_remote()
+ assert self._local_cas is not None, "Local CAS stub was not initialized"
+ return self._local_cas
+
def get_bytestream(self) -> bytestream_pb2_grpc.ByteStreamStub:
self._initialize_remote()
assert self._bytestream is not None, "Bytestream stub was not initialized"
@@ -618,19 +634,20 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
self.logger.debug("GetReference")
response = buildstream_pb2.GetReferenceResponse()
+ request = remote_execution_pb2.FindMissingBlobsRequest()
+ d = request.blob_digests.add()
+ d.CopyFrom(request.key)
+
try:
- tree = self.cas_cache.resolve_ref(request.key)
- try:
- self.cas_cache.update_tree_mtime(tree)
- except FileNotFoundError:
+ ref = self.cas.FindMissingBlobs(request)
+ except grpc.RpcError as err:
+ context.set_code(err.code())
+ if err.code() == grpc.StatusCode.NOT_FOUND:
self.cas_cache.remove_ref(request.key)
- context.set_code(grpc.StatusCode.NOT_FOUND)
- return response
+ return response
- response.digest.hash = tree.hash
- response.digest.size_bytes = tree.size_bytes
- except FileNotFoundError:
- context.set_code(grpc.StatusCode.NOT_FOUND)
+ response.digest.hash = ref.hash
+ response.digest.size_bytes = ref.size_bytes
return response
@@ -661,6 +678,7 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
def __init__(self, remote, root, cas_cache, *, update_cas=True):
super().__init__()
self.cas = remote.get_cas()
+ self.local_cas = remote.get_local_cas()
self.cas_cache = cas_cache
self.artifactdir = os.path.join(root, 'artifacts', 'refs')
self.update_cas = update_cas
@@ -677,6 +695,8 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
with open(artifact_path, 'rb') as f:
artifact.ParseFromString(f.read())
+ os.utime(artifact_path)
+
# Artifact-only servers will not have blobs on their system,
# so we can't reasonably update their mtimes. Instead, we exit
# early, and let the CAS server deal with its blobs.
@@ -695,28 +715,42 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
try:
if str(artifact.files):
- self.cas_cache.update_tree_mtime(artifact.files)
+ request = local_cas_pb2.FetchTreeRequest()
+ request.root_digest.CopyFrom(artifact.files)
+ request.fetch_file_blobs = True
+ self.local_cas.FetchTree(request)
if str(artifact.buildtree):
- # buildtrees might not be there
try:
- self.cas_cache.update_tree_mtime(artifact.buildtree)
- except FileNotFoundError:
- pass
+ request = local_cas_pb2.FetchTreeRequest()
+ request.root_digest.CopyFrom(artifact.buildtree)
+ request.fetch_file_blobs = True
+ self.local_cas.FetchTree(request)
+ except grpc.RpcError as err:
+ # buildtrees might not be there
+ if err.code() != grpc.StatusCode.NOT_FOUND:
+ raise
if str(artifact.public_data):
- os.utime(self.cas_cache.object_path(artifact.public_data))
+ request = remote_execution_pb2.FindMissingBlobsRequest()
+ d = request.blob_digests.add()
+ d.CopyFrom(artifact.public_data)
+ self.cas.FindMissingBlobs(request)
+ request = remote_execution_pb2.FindMissingBlobsRequest()
for log_file in artifact.logs:
- os.utime(self.cas_cache.object_path(log_file.digest))
-
- except FileNotFoundError:
- os.unlink(artifact_path)
- context.abort(grpc.StatusCode.NOT_FOUND,
- "Artifact files incomplete")
- except DecodeError:
- context.abort(grpc.StatusCode.NOT_FOUND,
- "Artifact files not valid")
+ d = request.blob_digests.add()
+ d.CopyFrom(log_file.digest)
+ self.cas.FindMissingBlobs(request)
+
+ except grpc.RpcError as err:
+ if err.code() == grpc.StatusCode.NOT_FOUND:
+ os.unlink(artifact_path)
+ context.abort(grpc.StatusCode.NOT_FOUND,
+ "Artifact files incomplete")
+ else:
+ context.abort(grpc.StatusCode.NOT_FOUND,
+ "Artifact files not valid")
return artifact
[buildstream] 11/19: workspaces.py: Make workspace config
group-readable
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cdbbe11baf3cc3d51fbc9159e52d4b6670225a94
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Nov 5 16:40:32 2019 +0000
workspaces.py: Make workspace config group-readable
---
src/buildstream/_workspaces.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/src/buildstream/_workspaces.py b/src/buildstream/_workspaces.py
index f9023dc..34ca92c 100644
--- a/src/buildstream/_workspaces.py
+++ b/src/buildstream/_workspaces.py
@@ -18,6 +18,7 @@
# Tristan Maat <tr...@codethink.co.uk>
import os
+import stat
from . import utils
from . import _yaml
@@ -128,6 +129,17 @@ class WorkspaceProject():
def write(self):
os.makedirs(self._directory, exist_ok=True)
_yaml.roundtrip_dump(self.to_dict(), self.get_filename())
+ # _yaml.roundtrip_dump() will create a file with very tight
+ # permissions (600). This isn't necessary here, and actively
+ # problematic when we're staging workspaces in a userchroot
+ # environment, since we won't be able to to read the file as a
+ # different user.
+ os.chmod(
+ self.get_filename(),
+ stat.S_IRUSR |
+ stat.S_IWUSR |
+ stat.S_IRGRP,
+ )
# get_filename()
#
[buildstream] 04/19: casserver.py: Proxy CAS requests to
buildbox-casd
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2a02568e0cbce765eb5d497afdb38df6d04d826a
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Oct 15 17:44:46 2019 +0100
casserver.py: Proxy CAS requests to buildbox-casd
---
src/buildstream/_cas/casserver.py | 274 +++++++++++---------------------------
tests/testutils/artifactshare.py | 4 +-
2 files changed, 77 insertions(+), 201 deletions(-)
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index 4f07639..d1bef68 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -36,8 +36,7 @@ from google.protobuf.message import DecodeError
import click
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
-from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
-from .._protos.google.rpc import code_pb2
+from .._protos.google.bytestream import bytestream_pb2_grpc
from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
artifact_pb2, artifact_pb2_grpc, source_pb2, source_pb2_grpc
@@ -49,6 +48,66 @@ from .. import utils
_MAX_PAYLOAD_BYTES = 1024 * 1024
+# CASRemote:
+#
+# A class that handles connections to a CAS remote - this is a (very)
+# slimmed down version of BuildStream's CASRemote.
+#
+class CASRemote:
+ def __init__(self, url: str):
+ self._url = url
+
+ self._bytestream = None
+ self._cas = None
+
+ # FIXME: We should allow setting up a secure channel. This
+ # isn't currently required, since we will only proxy to a
+ # process on the same host, but if we ever allow proxying to
+ # external services this will need to change.
+ self._channel = None
+
+ def _initialize_remote(self):
+ if self._channel:
+ assert self._cas and self._bytestream, "Stubs seem to have disappeared"
+ return
+ assert not (self._cas or self._bytestream), "Our cas/bytestream stubs should not have been set"
+
+ # Set up the remote channel
+ self._channel = grpc.insecure_channel(self._url)
+
+ # Assert that we support all capabilities we need
+ capabilities = remote_execution_pb2_grpc.CapabilitiesStub(self._channel)
+ start_wait = time.time()
+ while True:
+ try:
+ capabilities.GetCapabilities(remote_execution_pb2.GetCapabilitiesRequest())
+ break
+ except grpc.RpcError as e:
+ if e.code() == grpc.StatusCode.UNAVAILABLE:
+ # If connecting to casd, it may not be ready yet,
+ # try again after a 10ms delay, but don't wait for
+ # more than 15s
+ if time.time() < start_wait + 15:
+ time.sleep(1 / 100)
+ continue
+
+ raise
+
+ # Set up the RPC stubs
+ self._bytestream = bytestream_pb2_grpc.ByteStreamStub(self._channel)
+ self._cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self._channel)
+
+ def get_cas(self) -> remote_execution_pb2_grpc.ContentAddressableStorageStub:
+ self._initialize_remote()
+ assert self._cas is not None, "CAS stub was not initialized"
+ return self._cas
+
+ def get_bytestream(self) -> bytestream_pb2_grpc.ByteStreamStub:
+ self._initialize_remote()
+ assert self._bytestream is not None, "Bytestream stub was not initialized"
+ return self._bytestream
+
+
# CASCache:
#
# A slimmed down version of `buildstream._cas.cascache.CASCache` -
@@ -378,6 +437,7 @@ def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.WA
cas_runner = CASdRunner(os.path.abspath(repo), cache_quota=quota)
cas_runner.start_casd()
cas_cache = CASCache(os.path.abspath(repo))
+ cas = CASRemote('unix:' + cas_runner.get_socket_path())
try:
root = os.path.abspath(repo)
@@ -418,7 +478,6 @@ def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.WA
yield server
finally:
- cas.release_resources()
cas_runner.stop()
@@ -489,221 +548,41 @@ def server_main(repo, port, server_key, server_cert, client_certs, enable_push,
class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
- def __init__(self, cas, *, enable_push):
+ def __init__(self, remote, *, enable_push):
super().__init__()
- self.cas = cas
+ self.bytestream = remote.get_bytestream()
self.enable_push = enable_push
self.logger = logging.getLogger("casserver")
def Read(self, request, context):
self.logger.info("Read")
- resource_name = request.resource_name
- client_digest = _digest_from_download_resource_name(resource_name)
- if client_digest is None:
- context.set_code(grpc.StatusCode.NOT_FOUND)
- return
-
- if request.read_offset > client_digest.size_bytes:
- context.set_code(grpc.StatusCode.OUT_OF_RANGE)
- return
-
- try:
- with open(self.cas.objpath(client_digest), 'rb') as f:
- if os.fstat(f.fileno()).st_size != client_digest.size_bytes:
- context.set_code(grpc.StatusCode.NOT_FOUND)
- return
-
- os.utime(f.fileno())
-
- if request.read_offset > 0:
- f.seek(request.read_offset)
-
- remaining = client_digest.size_bytes - request.read_offset
- while remaining > 0:
- chunk_size = min(remaining, _MAX_PAYLOAD_BYTES)
- remaining -= chunk_size
-
- response = bytestream_pb2.ReadResponse()
- # max. 64 kB chunks
- response.data = f.read(chunk_size)
- yield response
- except FileNotFoundError:
- context.set_code(grpc.StatusCode.NOT_FOUND)
+ return self.bytestream.Read(request)
def Write(self, request_iterator, context):
self.logger.info("Write")
- response = bytestream_pb2.WriteResponse()
-
- if not self.enable_push:
- context.set_code(grpc.StatusCode.PERMISSION_DENIED)
- return response
-
- offset = 0
- finished = False
- resource_name = None
- with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
- for request in request_iterator:
- if finished or request.write_offset != offset:
- context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
- return response
-
- if resource_name is None:
- # First request
- resource_name = request.resource_name
- client_digest = _digest_from_upload_resource_name(resource_name)
- if client_digest is None:
- context.set_code(grpc.StatusCode.NOT_FOUND)
- return response
-
- while True:
- if client_digest.size_bytes == 0:
- break
-
- try:
- os.posix_fallocate(out.fileno(), 0, client_digest.size_bytes)
- break
- except OSError as e:
- # Multiple upload can happen in the same time
- if e.errno != errno.ENOSPC:
- raise
-
- elif request.resource_name:
- # If it is set on subsequent calls, it **must** match the value of the first request.
- if request.resource_name != resource_name:
- context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
- return response
-
- if (offset + len(request.data)) > client_digest.size_bytes:
- context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
- return response
-
- out.write(request.data)
- offset += len(request.data)
- if request.finish_write:
- if client_digest.size_bytes != offset:
- context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
- return response
- out.flush()
-
- try:
- digest = self.cas.add_object(path=out.name, link_directly=True)
- except CASCacheError as e:
- if e.reason == "cache-too-full":
- context.set_code(grpc.StatusCode.RESOURCE_EXHAUSTED)
- else:
- context.set_code(grpc.StatusCode.INTERNAL)
- return response
-
- if digest.hash != client_digest.hash:
- context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
- return response
-
- finished = True
-
- assert finished
-
- response.committed_size = offset
- return response
+ return self.bytestream.Write(request_iterator)
class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddressableStorageServicer):
- def __init__(self, cas, *, enable_push):
+ def __init__(self, remote, *, enable_push):
super().__init__()
- self.cas = cas
+ self.cas = remote.get_cas()
self.enable_push = enable_push
self.logger = logging.getLogger("casserver")
def FindMissingBlobs(self, request, context):
self.logger.info("FindMissingBlobs")
- response = remote_execution_pb2.FindMissingBlobsResponse()
- for digest in request.blob_digests:
- objpath = self.cas.objpath(digest)
- try:
- os.utime(objpath)
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
-
- d = response.missing_blob_digests.add()
- d.hash = digest.hash
- d.size_bytes = digest.size_bytes
-
- return response
+ self.logger.debug(request.blob_digests)
+ return self.cas.FindMissingBlobs(request)
def BatchReadBlobs(self, request, context):
self.logger.info("BatchReadBlobs")
- self.logger.debug(request.digests)
- response = remote_execution_pb2.BatchReadBlobsResponse()
- batch_size = 0
-
- for digest in request.digests:
- batch_size += digest.size_bytes
- if batch_size > _MAX_PAYLOAD_BYTES:
- context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
- return response
-
- blob_response = response.responses.add()
- blob_response.digest.hash = digest.hash
- blob_response.digest.size_bytes = digest.size_bytes
- try:
- objpath = self.cas.objpath(digest)
- with open(objpath, 'rb') as f:
- if os.fstat(f.fileno()).st_size != digest.size_bytes:
- blob_response.status.code = code_pb2.NOT_FOUND
- continue
-
- os.utime(f.fileno())
-
- blob_response.data = f.read(digest.size_bytes)
- except FileNotFoundError:
- blob_response.status.code = code_pb2.NOT_FOUND
-
- return response
+ return self.cas.BatchReadBlobs(request)
def BatchUpdateBlobs(self, request, context):
self.logger.info("BatchUpdateBlobs")
self.logger.debug([request.digest for request in request.requests])
- response = remote_execution_pb2.BatchUpdateBlobsResponse()
-
- if not self.enable_push:
- context.set_code(grpc.StatusCode.PERMISSION_DENIED)
- return response
-
- batch_size = 0
-
- for blob_request in request.requests:
- digest = blob_request.digest
-
- batch_size += digest.size_bytes
- if batch_size > _MAX_PAYLOAD_BYTES:
- context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
- return response
-
- blob_response = response.responses.add()
- blob_response.digest.hash = digest.hash
- blob_response.digest.size_bytes = digest.size_bytes
-
- if len(blob_request.data) != digest.size_bytes:
- blob_response.status.code = code_pb2.FAILED_PRECONDITION
- continue
-
- with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
- out.write(blob_request.data)
- out.flush()
-
- try:
- server_digest = self.cas.add_object(path=out.name)
- except CASCacheError as e:
- if e.reason == "cache-too-full":
- blob_response.status.code = code_pb2.RESOURCE_EXHAUSTED
- else:
- blob_response.status.code = code_pb2.INTERNAL
- continue
-
- if server_digest.hash != digest.hash:
- blob_response.status.code = code_pb2.FAILED_PRECONDITION
-
- return response
+ return self.cas.BatchUpdateBlobs(request)
class _CapabilitiesServicer(remote_execution_pb2_grpc.CapabilitiesServicer):
@@ -728,9 +607,9 @@ class _CapabilitiesServicer(remote_execution_pb2_grpc.CapabilitiesServicer):
class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
- def __init__(self, cas, cas_cache, *, enable_push):
+ def __init__(self, remote, cas_cache, *, enable_push):
super().__init__()
- self.cas = cas
+ self.cas = remote.get_cas()
self.cas_cache = cas_cache
self.enable_push = enable_push
self.logger = logging.getLogger("casserver")
@@ -779,13 +658,12 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
- def __init__(self, cas, root, cas_cache, *, update_cas=True):
+ def __init__(self, remote, root, cas_cache, *, update_cas=True):
super().__init__()
- self.cas = cas
+ self.cas = remote.get_cas()
self.cas_cache = cas_cache
self.artifactdir = os.path.join(root, 'artifacts', 'refs')
self.update_cas = update_cas
- os.makedirs(artifactdir, exist_ok=True)
self.logger = logging.getLogger("casserver")
def GetArtifact(self, request, context):
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 18ecc5e..d86cafa 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -39,11 +39,9 @@ class ArtifactShare():
# in tests as a remote artifact push/pull configuration
#
self.repodir = os.path.join(self.directory, 'repo')
- os.makedirs(self.repodir)
- self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs')
- os.makedirs(self.artifactdir)
self.cas = CASCache(self.repodir, casd=casd)
+ self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs')
self.quota = quota
self.index_only = index_only
[buildstream] 19/19: tar.py: Make directories with incorrect
permissions traversable
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0a8126cdc7e234d5dd82fb806cb0364188effb31
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Nov 12 11:04:25 2019 +0000
tar.py: Make directories with incorrect permissions traversable
---
src/buildstream/plugins/sources/tar.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py
index ac80177..a0682e0 100644
--- a/src/buildstream/plugins/sources/tar.py
+++ b/src/buildstream/plugins/sources/tar.py
@@ -75,8 +75,12 @@ class ReadableTarInfo(tarfile.TarInfo):
"""
@property
def mode(self):
- # ensure file is readable by owner and group
- return self.__permission | 0o440
+ # ensure file is readable by owner and group, and executable
+ # (=traversable) if it's a directory.
+ if self.isdir():
+ return self.__permission | 0o550
+ else:
+ return self.__permission | 0o440
@mode.setter
def mode(self, permission):
[buildstream] 12/19: element.py: Make build directory a proper
utils._tempdir
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5822e0f7b8e79e306f88b93338e7325feacefaae
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Thu Nov 7 16:16:30 2019 +0000
element.py: Make build directory a proper utils._tempdir
---
src/buildstream/element.py | 9 +--------
tests/testutils/artifactshare.py | 2 --
2 files changed, 1 insertion(+), 10 deletions(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 7bb8bc0..f94dcd1 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1644,13 +1644,8 @@ class Element(Plugin):
# Explicitly clean it up, keep the build dir around if exceptions are raised
os.makedirs(context.builddir, exist_ok=True)
- rootdir = tempfile.mkdtemp(prefix="{}-".format(self.normal_name), dir=context.builddir)
-
- # Cleanup the build directory on explicit SIGTERM
- def cleanup_rootdir():
- utils._force_rmtree(rootdir)
- with _signals.terminator(cleanup_rootdir), \
+ with utils._group_tempdir(prefix="{}-".format(self.normal_name), dir=context.builddir) as rootdir, \
self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # noqa
# Let the sandbox know whether the buildtree will be required.
@@ -1703,8 +1698,6 @@ class Element(Plugin):
raise
else:
return self._cache_artifact(rootdir, sandbox, collect)
- finally:
- cleanup_rootdir()
def _cache_artifact(self, rootdir, sandbox, collect):
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index ba02d39..52e2708 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -195,8 +195,6 @@ class ArtifactShare():
self.cas.release_resources()
- shutil.rmtree(self.directory)
-
# create_artifact_share()
#
[buildstream] 14/19: cascache.py: Set up socket path via a symlink
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bcad8f400bf4f20faef87e3e58d4602dd3310cab
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Mon Nov 4 13:22:55 2019 +0000
cascache.py: Set up socket path via a symlink
This is necessary to allow using buildbox-run with userchroot in the
near future, since currently only the owner of the BuildStream process
can access the CASD socket, but the buildbox-casd binary will need to
be setuid' to another user.
This gets around this limitation by allowing the group to access a
symlink, which in turn should point to a directory owned by the CASD
user.
---
src/buildstream/_cas/cascache.py | 64 +++++++++++++++++++++++++++++++++++++---
1 file changed, 60 insertions(+), 4 deletions(-)
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 11f15bd..bcd835e 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -24,6 +24,7 @@ import stat
import contextlib
import ctypes
import multiprocessing
+import random
import shutil
import signal
import subprocess
@@ -84,10 +85,7 @@ class CASCache():
self._cache_usage_monitor_forbidden = False
if casd:
- # Place socket in global/user temporary directory to avoid hitting
- # the socket path length limit.
- self._casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream')
- self._casd_socket_path = os.path.join(self._casd_socket_tempdir, 'casd.sock')
+ self._casd_socket_path = self._make_socket_path(path)
casd_args = [utils.get_host_tool('buildbox-casd')]
casd_args.append('--bind=unix:' + self._casd_socket_path)
@@ -116,6 +114,64 @@ class CASCache():
else:
self._casd_process = None
+ # _make_socket_path()
+ #
+ # Create a path to the CASD socket, ensuring that we don't exceed
+ # the socket path limit.
+ #
+ # Note that we *may* exceed the path limit if the python-chosen
+ # tmpdir path is very long, though this should be /tmp.
+ #
+ # Args:
+ # path (str): The root directory for the CAS repository.
+ #
+ # Returns:
+ # (str) - The path to the CASD socket.
+ #
+ def _make_socket_path(self, path):
+ self._casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream')
+ # mkdtemp will create this directory in the "most secure"
+ # way. This translates to "u+rwx,go-rwx".
+ #
+ # This is a good thing, generally, since it prevents us
+ # from leaking sensitive information to other users, but
+ # it's a problem for the workflow for userchroot, since
+ # the setuid casd binary will not share a uid with the
+ # user creating the tempdir.
+ #
+ # Instead, we chmod the directory 750, and only place a
+ # symlink to the CAS directory in here, which will allow the
+ # CASD process RWX access to a directory without leaking build
+ # information.
+ os.chmod(
+ self._casd_socket_tempdir,
+ stat.S_IRUSR |
+ stat.S_IWUSR |
+ stat.S_IXUSR |
+ stat.S_IRGRP |
+ stat.S_IXGRP,
+ )
+
+ os.symlink(path, os.path.join(self._casd_socket_tempdir, "cas"))
+ # FIXME: There is a potential race condition here; if multiple
+ # instances happen to create the same socket path, at least
+ # one will try to talk to the same server as us.
+ #
+ # There's no real way to avoid this from our side; we'd need
+ # buildbox-casd to tell us that it could not create a fresh
+ # socket.
+ #
+ # We could probably make this even safer by including some
+ # thread/process-specific information, but we're not really
+ # supporting this use case anyway; it's mostly here fore
+ # testing, and to help more gracefully handle the situation.
+ #
+ # Note: this uses the same random string generation principle
+ # as cpython, so this is probably a safe file name.
+ socket_name = "casserver-{}.sock".format(
+ "".join(random.choices("abcdefghijklmnopqrstuvwxyz0123456789_", k=8)))
+ return os.path.join(self._casd_socket_tempdir, "cas", socket_name)
+
def __getstate__(self):
state = self.__dict__.copy()
[buildstream] 13/19: Make roundtrip_dump allow group permissions
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 288bd39c5ecd74a5817f86159f0a2153515d3bd3
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Thu Nov 7 16:16:51 2019 +0000
Make roundtrip_dump allow group permissions
---
src/buildstream/_yaml.pyx | 6 ++++--
src/buildstream/utils.py | 13 ++++++++++++-
tests/artifactcache/junctions.py | 2 +-
3 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/src/buildstream/_yaml.pyx b/src/buildstream/_yaml.pyx
index 797e10d..1658a70 100644
--- a/src/buildstream/_yaml.pyx
+++ b/src/buildstream/_yaml.pyx
@@ -486,14 +486,16 @@ def roundtrip_load_data(contents, *, filename=None):
# Args:
# contents (Mapping or list): The content to write out as YAML.
# file (any): The file to write to
+# group_accessible (bool): Whether the resulting file should be group accessible.
#
-def roundtrip_dump(contents, file=None):
+def roundtrip_dump(contents, file=None, group_accessible=False):
with ExitStack() as stack:
if type(file) is str:
from . import utils
- f = stack.enter_context(utils.save_file_atomic(file, 'w'))
+ f = stack.enter_context(utils.save_file_atomic(file, 'w', group_accessible=group_accessible))
elif hasattr(file, 'write'):
f = file
else:
f = sys.stdout
yaml.round_trip_dump(contents, f, Dumper=HardlineDumper)
+
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index e9f0fb7..9ca5a2b 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -567,7 +567,8 @@ def save_file_atomic(filename: str,
newline: Optional[str] = None,
closefd: bool = True,
opener: Optional[Callable[[str, int], int]] = None,
- tempdir: Optional[str] = None) -> Iterator[IO]:
+ tempdir: Optional[str] = None,
+ group_accessible: bool = False) -> Iterator[IO]:
"""Save a file with a temporary name and rename it into place when ready.
This is a context manager which is meant for saving data to files.
@@ -620,6 +621,16 @@ def save_file_atomic(filename: str,
# This operation is atomic, at least on platforms we care about:
# https://bugs.python.org/issue8828
os.replace(tempname, filename)
+ if group_accessible:
+ os.chmod(
+ filename,
+ stat.S_IWUSR |
+ stat.S_IRUSR |
+ stat.S_IXUSR |
+ stat.S_IWGRP |
+ stat.S_IRGRP |
+ stat.S_IXGRP,
+ )
except Exception:
cleanup_tempfile()
raise
diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py
index dab69ea..32788ef 100644
--- a/tests/artifactcache/junctions.py
+++ b/tests/artifactcache/junctions.py
@@ -24,7 +24,7 @@ def project_set_artifacts(project, url):
'url': url,
'push': True
}
- _yaml.roundtrip_dump(project_config.strip_node_info(), file=project_conf_file)
+ _yaml.roundtrip_dump(project_config.strip_node_info(), file=project_conf_file, group_accessible=True)
@pytest.mark.datafiles(DATA_DIR)
[buildstream] 16/19: plugins/sources/tar.py: Ensure read
permissions for groups as well
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 397ddcfe7a728aec06307ee24f7c8be05130f490
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Fri Nov 8 12:24:43 2019 +0000
plugins/sources/tar.py: Ensure read permissions for groups as well
---
src/buildstream/plugins/sources/tar.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py
index 702b7ba..ac80177 100644
--- a/src/buildstream/plugins/sources/tar.py
+++ b/src/buildstream/plugins/sources/tar.py
@@ -69,14 +69,14 @@ from ._downloadablefilesource import DownloadableFileSource
class ReadableTarInfo(tarfile.TarInfo):
"""
The goal is to override `TarFile`'s `extractall` semantics by ensuring that on extraction, the
- files are readable by the owner of the file. This is done by overriding the accessor for the
+ files are readable by the owners of the file. This is done by overriding the accessor for the
`mode` attribute in `TarInfo`, the class that encapsulates the internal meta-data of the tarball,
so that the owner-read bit is always set.
"""
@property
def mode(self):
- # ensure file is readable by owner
- return self.__permission | 0o400
+ # ensure file is readable by owner and group
+ return self.__permission | 0o440
@mode.setter
def mode(self, permission):
[buildstream] 08/19: WIP: Temporarily disable the ability to link
in CasBasedDirectory
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 44bd2d72172297a60d874a9ff47259007372d18c
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Wed Nov 6 13:38:07 2019 +0000
WIP: Temporarily disable the ability to link in CasBasedDirectory
---
src/buildstream/storage/_filebaseddirectory.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/buildstream/storage/_filebaseddirectory.py b/src/buildstream/storage/_filebaseddirectory.py
index 07c23c1..cb4206a 100644
--- a/src/buildstream/storage/_filebaseddirectory.py
+++ b/src/buildstream/storage/_filebaseddirectory.py
@@ -86,6 +86,8 @@ class FileBasedDirectory(Directory):
from ._casbaseddirectory import CasBasedDirectory # pylint: disable=cyclic-import
+ can_link = False
+
if isinstance(external_pathspec, CasBasedDirectory):
if can_link and not update_mtime:
actionfunc = utils.safe_link
[buildstream] 06/19: Remove newly unused API surfaces in CASCache
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c424e99d4315c2d12d7bc803c9f894f8ddbb5cea
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Wed Oct 16 17:14:15 2019 +0100
Remove newly unused API surfaces in CASCache
This also involves a number of changes to tests and other parts of the
codebase since they were hacking about wit API that shouldn't have
existed.
---
src/buildstream/_artifactcache.py | 6 +-
src/buildstream/_basecache.py | 54 ++++++++++++-
src/buildstream/_cas/cascache.py | 165 +-------------------------------------
src/buildstream/_exceptions.py | 9 +++
tests/sourcecache/fetch.py | 9 +--
tests/sourcecache/push.py | 6 +-
tests/testutils/artifactshare.py | 14 +++-
7 files changed, 84 insertions(+), 179 deletions(-)
diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py
index d9112cd..91225ea 100644
--- a/src/buildstream/_artifactcache.py
+++ b/src/buildstream/_artifactcache.py
@@ -21,7 +21,7 @@ import os
import grpc
from ._basecache import BaseCache
-from ._exceptions import ArtifactError, CASError, CASCacheError, CASRemoteError
+from ._exceptions import ArtifactError, CASError, CacheError, CASRemoteError
from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
artifact_pb2, artifact_pb2_grpc
@@ -203,8 +203,8 @@ class ArtifactCache(BaseCache):
#
def remove(self, ref):
try:
- self.cas.remove(ref, basedir=self.artifactdir)
- except CASCacheError as e:
+ self._remove_ref(ref, self.artifactdir)
+ except CacheError as e:
raise ArtifactError("{}".format(e)) from e
# diff():
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py
index fc2e924..69da0f9 100644
--- a/src/buildstream/_basecache.py
+++ b/src/buildstream/_basecache.py
@@ -17,6 +17,7 @@
# Raoul Hidalgo Charman <ra...@codethink.co.uk>
#
import os
+import errno
from fnmatch import fnmatch
from itertools import chain
from typing import TYPE_CHECKING
@@ -25,7 +26,7 @@ from . import utils
from . import _yaml
from ._cas import CASRemote
from ._message import Message, MessageType
-from ._exceptions import LoadError, RemoteError
+from ._exceptions import LoadError, RemoteError, CacheError
from ._remote import RemoteSpec, RemoteType
@@ -425,3 +426,54 @@ class BaseCache():
if not glob_expr or fnmatch(relative_path, glob_expr):
# Obtain the mtime (the time a file was last modified)
yield (os.path.getmtime(ref_path), relative_path)
+
+ # _remove_ref()
+ #
+ # Removes a ref.
+ #
+ # This also takes care of pruning away directories which can
+ # be removed after having removed the given ref.
+ #
+ # Args:
+ # ref (str): The ref to remove
+ # basedir (str): Path of base directory the ref is in
+ #
+ # Raises:
+ # (CASCacheError): If the ref didnt exist, or a system error
+ # occurred while removing it
+ #
+ def _remove_ref(self, ref, basedir):
+
+ # Remove the ref itself
+ refpath = os.path.join(basedir, ref)
+
+ try:
+ os.unlink(refpath)
+ except FileNotFoundError as e:
+ raise CacheError("Could not find ref '{}'".format(ref)) from e
+
+ # Now remove any leading directories
+
+ components = list(os.path.split(ref))
+ while components:
+ components.pop()
+ refdir = os.path.join(basedir, *components)
+
+ # Break out once we reach the base
+ if refdir == basedir:
+ break
+
+ try:
+ os.rmdir(refdir)
+ except FileNotFoundError:
+ # The parent directory did not exist, but it's
+ # parent directory might still be ready to prune
+ pass
+ except OSError as e:
+ if e.errno == errno.ENOTEMPTY:
+ # The parent directory was not empty, so we
+ # cannot prune directories beyond this point
+ break
+
+ # Something went wrong here
+ raise CacheError("System error while removing ref '{}': {}".format(ref, e)) from e
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 0227304..11f15bd 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -21,7 +21,6 @@
import itertools
import os
import stat
-import errno
import contextlib
import ctypes
import multiprocessing
@@ -75,7 +74,6 @@ class CASCache():
):
self.casdir = os.path.join(path, 'cas')
self.tmpdir = os.path.join(path, 'tmp')
- os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
os.makedirs(self.tmpdir, exist_ok=True)
@@ -179,9 +177,7 @@ class CASCache():
# Preflight check.
#
def preflight(self):
- headdir = os.path.join(self.casdir, 'refs', 'heads')
- objdir = os.path.join(self.casdir, 'objects')
- if not (os.path.isdir(headdir) and os.path.isdir(objdir)):
+ if not os.path.join(self.casdir, 'objects'):
raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir))
# has_open_grpc_channels():
@@ -216,21 +212,6 @@ class CASCache():
self._terminate_casd_process(messenger)
shutil.rmtree(self._casd_socket_tempdir)
- # contains():
- #
- # Check whether the specified ref is already available in the local CAS cache.
- #
- # Args:
- # ref (str): The ref to check
- #
- # Returns: True if the ref is in the cache, False otherwise
- #
- def contains(self, ref):
- refpath = self._refpath(ref)
-
- # This assumes that the repository doesn't have any dangling pointers
- return os.path.exists(refpath)
-
# contains_file():
#
# Check whether a digest corresponds to a file which exists in CAS
@@ -309,28 +290,6 @@ class CASCache():
fullpath = os.path.join(dest, symlinknode.name)
os.symlink(symlinknode.target, fullpath)
- # diff():
- #
- # Return a list of files that have been added or modified between
- # the refs described by ref_a and ref_b.
- #
- # Args:
- # ref_a (str): The first ref
- # ref_b (str): The second ref
- # subdir (str): A subdirectory to limit the comparison to
- #
- def diff(self, ref_a, ref_b):
- tree_a = self.resolve_ref(ref_a)
- tree_b = self.resolve_ref(ref_b)
-
- added = []
- removed = []
- modified = []
-
- self.diff_trees(tree_a, tree_b, added=added, removed=removed, modified=modified)
-
- return modified, removed, added
-
# pull_tree():
#
# Pull a single Tree rather than a ref.
@@ -457,74 +416,6 @@ class CASCache():
return utils._message_digest(root_directory)
- # set_ref():
- #
- # Create or replace a ref.
- #
- # Args:
- # ref (str): The name of the ref
- #
- def set_ref(self, ref, tree):
- refpath = self._refpath(ref)
- os.makedirs(os.path.dirname(refpath), exist_ok=True)
- with utils.save_file_atomic(refpath, 'wb', tempdir=self.tmpdir) as f:
- f.write(tree.SerializeToString())
-
- # resolve_ref():
- #
- # Resolve a ref to a digest.
- #
- # Args:
- # ref (str): The name of the ref
- # update_mtime (bool): Whether to update the mtime of the ref
- #
- # Returns:
- # (Digest): The digest stored in the ref
- #
- def resolve_ref(self, ref, *, update_mtime=False):
- refpath = self._refpath(ref)
-
- try:
- with open(refpath, 'rb') as f:
- if update_mtime:
- os.utime(refpath)
-
- digest = remote_execution_pb2.Digest()
- digest.ParseFromString(f.read())
- return digest
-
- except FileNotFoundError as e:
- raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
-
- # update_mtime()
- #
- # Update the mtime of a ref.
- #
- # Args:
- # ref (str): The ref to update
- #
- def update_mtime(self, ref):
- try:
- os.utime(self._refpath(ref))
- except FileNotFoundError as e:
- raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
-
- # remove():
- #
- # Removes the given symbolic ref from the repo.
- #
- # Args:
- # ref (str): A symbolic ref
- # basedir (str): Path of base directory the ref is in, defaults to
- # CAS refs heads
- #
- def remove(self, ref, *, basedir=None):
-
- if basedir is None:
- basedir = os.path.join(self.casdir, 'refs', 'heads')
- # Remove cache ref
- self._remove_ref(ref, basedir)
-
def update_tree_mtime(self, tree):
reachable = set()
self._reachable_refs_dir(reachable, tree, update_mtime=True)
@@ -702,60 +593,6 @@ class CASCache():
return os.path.join(log_dir, str(self._casd_start_time) + ".log")
- def _refpath(self, ref):
- return os.path.join(self.casdir, 'refs', 'heads', ref)
-
- # _remove_ref()
- #
- # Removes a ref.
- #
- # This also takes care of pruning away directories which can
- # be removed after having removed the given ref.
- #
- # Args:
- # ref (str): The ref to remove
- # basedir (str): Path of base directory the ref is in
- #
- # Raises:
- # (CASCacheError): If the ref didnt exist, or a system error
- # occurred while removing it
- #
- def _remove_ref(self, ref, basedir):
-
- # Remove the ref itself
- refpath = os.path.join(basedir, ref)
-
- try:
- os.unlink(refpath)
- except FileNotFoundError as e:
- raise CASCacheError("Could not find ref '{}'".format(ref)) from e
-
- # Now remove any leading directories
-
- components = list(os.path.split(ref))
- while components:
- components.pop()
- refdir = os.path.join(basedir, *components)
-
- # Break out once we reach the base
- if refdir == basedir:
- break
-
- try:
- os.rmdir(refdir)
- except FileNotFoundError:
- # The parent directory did not exist, but it's
- # parent directory might still be ready to prune
- pass
- except OSError as e:
- if e.errno == errno.ENOTEMPTY:
- # The parent directory was not empty, so we
- # cannot prune directories beyond this point
- break
-
- # Something went wrong here
- raise CASCacheError("System error while removing ref '{}': {}".format(ref, e)) from e
-
def _get_subdir(self, tree, subdir):
head, name = os.path.split(subdir)
if head:
diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py
index 947b831..319518f 100644
--- a/src/buildstream/_exceptions.py
+++ b/src/buildstream/_exceptions.py
@@ -275,6 +275,15 @@ class SandboxError(BstError):
# SourceCacheError
#
+# Raised when errors are encountered in either type of cache
+#
+class CacheError(BstError):
+ def __init__(self, message, detail=None, reason=None):
+ super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason)
+
+
+# SourceCacheError
+#
# Raised when errors are encountered in the source caches
#
class SourceCacheError(BstError):
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index a5863b8..48683d6 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -92,8 +92,7 @@ def test_source_fetch(cli, tmpdir, datafiles):
assert not element._source_cached()
source = list(element.sources())[0]
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not share.get_source_proto(source._get_source_name())
# Just check that we sensibly fetch and build the element
res = cli.run(project=project_dir, args=['build', element_name])
@@ -139,8 +138,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
assert not element._source_cached()
source = list(element.sources())[0]
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not share.get_source_proto(source._get_source_name())
assert not os.path.exists(os.path.join(cache_dir, 'sources'))
# Now check if it falls back to the source fetch method.
@@ -198,8 +196,7 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
assert not element._source_cached()
source = list(element.sources())[0]
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not share.get_artifact_proto(source._get_source_name())
# Just check that we sensibly fetch and build the element
res = cli.run(project=project_dir, args=['build', element_name])
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 406aeba..8c0ac06 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -98,8 +98,7 @@ def test_source_push_split(cli, tmpdir, datafiles):
source = list(element.sources())[0]
# check we don't have it in the current cache
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not index.get_source_proto(source._get_source_name())
# build the element, this should fetch and then push the source to the
# remote
@@ -156,8 +155,7 @@ def test_source_push(cli, tmpdir, datafiles):
source = list(element.sources())[0]
# check we don't have it in the current cache
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not share.get_source_proto(source._get_source_name())
# build the element, this should fetch and then push the source to the
# remote
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index d86cafa..ba02d39 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -11,7 +11,7 @@ from buildstream._cas import CASCache
from buildstream._cas.casserver import create_server
from buildstream._exceptions import CASError
from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
-from buildstream._protos.buildstream.v2 import artifact_pb2
+from buildstream._protos.buildstream.v2 import artifact_pb2, source_pb2
# ArtifactShare()
@@ -42,6 +42,7 @@ class ArtifactShare():
self.cas = CASCache(self.repodir, casd=casd)
self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs')
+ self.sourcedir = os.path.join(self.repodir, 'source_protos', 'refs')
self.quota = quota
self.index_only = index_only
@@ -127,6 +128,17 @@ class ArtifactShare():
return artifact_proto
+ def get_source_proto(self, source_name):
+ source_proto = source_pb2.Source()
+ source_path = os.path.join(self.sourcedir, source_name)
+
+ try:
+ with open(source_path, 'rb') as f:
+ source_proto.ParseFromString(f.read())
+ except FileNotFoundError:
+ return None
+ return source_proto
+
def get_cas_files(self, artifact_proto):
reachable = set()
[buildstream] 01/19: casserver.py: Add logging
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit abaa1d1ec6cf858de0eac48271b0bff3d5b71139
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Fri Oct 11 15:19:19 2019 +0100
casserver.py: Add logging
---
src/buildstream/_cas/casserver.py | 76 ++++++++++++++++++++++++++++++++++++---
1 file changed, 71 insertions(+), 5 deletions(-)
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index d424143..d5a29a3 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -19,6 +19,8 @@
from concurrent import futures
from contextlib import contextmanager
+from enum import Enum
+import logging
import os
import signal
import sys
@@ -41,12 +43,41 @@ from .._exceptions import CASError, CASCacheError
from .cascache import CASCache
-
# The default limit for gRPC messages is 4 MiB.
# Limit payload to 1 MiB to leave sufficient headroom for metadata.
_MAX_PAYLOAD_BYTES = 1024 * 1024
+# LogLevel():
+#
+# Represents the buildbox-casd log level.
+#
+class LogLevel(Enum):
+ WARNING = "warning"
+ INFO = "info"
+ TRACE = "trace"
+
+ @classmethod
+ def get_logging_equivalent(cls, level: 'LogLevel') -> int:
+ equivalents = {
+ cls.WARNING: logging.WARNING,
+ cls.INFO: logging.INFO,
+ cls.TRACE: logging.DEBUG
+ }
+
+ # Yes, logging.WARNING/INFO/DEBUG are ints
+ # I also don't know why
+ return equivalents[level]
+
+
+class ClickLogLevel(click.Choice):
+ def __init__(self):
+ super().__init__([m.lower() for m in LogLevel._member_names_]) # pylint: disable=no-member
+
+ def convert(self, value, param, ctx):
+ return LogLevel(super().convert(value, param, ctx))
+
+
# create_server():
#
# Create gRPC CAS artifact server as specified in the Remote Execution API.
@@ -56,8 +87,14 @@ _MAX_PAYLOAD_BYTES = 1024 * 1024
# enable_push (bool): Whether to allow blob uploads and artifact updates
# index_only (bool): Whether to store CAS blobs or only artifacts
#
-@contextmanager
-def create_server(repo, *, enable_push, quota, index_only):
+@contextlib.contextmanager
+def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.WARNING):
+ logger = logging.getLogger('casserver')
+ logger.setLevel(LogLevel.get_logging_equivalent(log_level))
+ handler = logging.StreamHandler(sys.stderr)
+ handler.setLevel(LogLevel.get_logging_equivalent(log_level))
+ logger.addHandler(handler)
+
cas = CASCache(os.path.abspath(repo), cache_quota=quota, protect_session_blobs=False)
try:
@@ -113,9 +150,11 @@ def create_server(repo, *, enable_push, quota, index_only):
help="Maximum disk usage in bytes")
@click.option('--index-only', is_flag=True,
help="Only provide the BuildStream artifact and source services (\"index\"), not the CAS (\"storage\")")
+@click.option('--log-level', type=ClickLogLevel(),
+ help="The log level to launch with")
@click.argument('repo')
def server_main(repo, port, server_key, server_cert, client_certs, enable_push,
- quota, index_only):
+ quota, index_only, log_level):
# Handle SIGTERM by calling sys.exit(0), which will raise a SystemExit exception,
# properly executing cleanup code in `finally` clauses and context managers.
# This is required to terminate buildbox-casd on SIGTERM.
@@ -124,7 +163,8 @@ def server_main(repo, port, server_key, server_cert, client_certs, enable_push,
with create_server(repo,
quota=quota,
enable_push=enable_push,
- index_only=index_only) as server:
+ index_only=index_only,
+ log_level=log_level) as server:
use_tls = bool(server_key)
@@ -170,8 +210,10 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
super().__init__()
self.cas = cas
self.enable_push = enable_push
+ self.logger = logging.getLogger("casserver")
def Read(self, request, context):
+ self.logger.info("Read")
resource_name = request.resource_name
client_digest = _digest_from_download_resource_name(resource_name)
if client_digest is None:
@@ -206,6 +248,7 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
context.set_code(grpc.StatusCode.NOT_FOUND)
def Write(self, request_iterator, context):
+ self.logger.info("Write")
response = bytestream_pb2.WriteResponse()
if not self.enable_push:
@@ -285,8 +328,10 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
super().__init__()
self.cas = cas
self.enable_push = enable_push
+ self.logger = logging.getLogger("casserver")
def FindMissingBlobs(self, request, context):
+ self.logger.info("FindMissingBlobs")
response = remote_execution_pb2.FindMissingBlobsResponse()
for digest in request.blob_digests:
objpath = self.cas.objpath(digest)
@@ -303,6 +348,8 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
return response
def BatchReadBlobs(self, request, context):
+ self.logger.info("BatchReadBlobs")
+ self.logger.debug(request.digests)
response = remote_execution_pb2.BatchReadBlobsResponse()
batch_size = 0
@@ -331,6 +378,8 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
return response
def BatchUpdateBlobs(self, request, context):
+ self.logger.info("BatchUpdateBlobs")
+ self.logger.debug([request.digest for request in request.requests])
response = remote_execution_pb2.BatchUpdateBlobsResponse()
if not self.enable_push:
@@ -375,7 +424,11 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
class _CapabilitiesServicer(remote_execution_pb2_grpc.CapabilitiesServicer):
+ def __init__(self):
+ self.logger = logging.getLogger("casserver")
+
def GetCapabilities(self, request, context):
+ self.logger.info("GetCapabilities")
response = remote_execution_pb2.ServerCapabilities()
cache_capabilities = response.cache_capabilities
@@ -396,8 +449,10 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
super().__init__()
self.cas = cas
self.enable_push = enable_push
+ self.logger = logging.getLogger("casserver")
def GetReference(self, request, context):
+ self.logger.debug("GetReference")
response = buildstream_pb2.GetReferenceResponse()
try:
@@ -417,6 +472,7 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
return response
def UpdateReference(self, request, context):
+ self.logger.debug("UpdateReference")
response = buildstream_pb2.UpdateReferenceResponse()
if not self.enable_push:
@@ -429,6 +485,7 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
return response
def Status(self, request, context):
+ self.logger.debug("Status")
response = buildstream_pb2.StatusResponse()
response.allow_updates = self.enable_push
@@ -444,8 +501,11 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
self.artifactdir = artifactdir
self.update_cas = update_cas
os.makedirs(artifactdir, exist_ok=True)
+ self.logger = logging.getLogger("casserver")
def GetArtifact(self, request, context):
+ self.logger.info("GetArtifact")
+ self.logger.debug(request.cache_key)
artifact_path = os.path.join(self.artifactdir, request.cache_key)
if not os.path.exists(artifact_path):
context.abort(grpc.StatusCode.NOT_FOUND, "Artifact proto not found")
@@ -498,6 +558,8 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
return artifact
def UpdateArtifact(self, request, context):
+ self.logger.info("UpdateArtifact")
+ self.logger.debug(request.cache_key)
artifact = request.artifact
if self.update_cas:
@@ -522,6 +584,7 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
return artifact
def ArtifactStatus(self, request, context):
+ self.logger.info("ArtifactStatus")
return artifact_pb2.ArtifactStatusResponse()
def _check_directory(self, name, digest, context):
@@ -530,9 +593,11 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
with open(self.cas.objpath(digest), 'rb') as f:
directory.ParseFromString(f.read())
except FileNotFoundError:
+ self.logger.warning("Artifact %s specified but no files found (%s)", name, self.cas.objpath(digest))
context.abort(grpc.StatusCode.FAILED_PRECONDITION,
"Artifact {} specified but no files found".format(name))
except DecodeError:
+ self.logger.warning("Artifact %s specified but directory not found (%s)", name, self.cas.objpath(digest))
context.abort(grpc.StatusCode.FAILED_PRECONDITION,
"Artifact {} specified but directory not found".format(name))
@@ -557,6 +622,7 @@ class _BuildStreamCapabilitiesServicer(buildstream_pb2_grpc.CapabilitiesServicer
class _SourceServicer(source_pb2_grpc.SourceServiceServicer):
def __init__(self, sourcedir):
self.sourcedir = sourcedir
+ self.logger = logging.getLogger("casserver")
def GetSource(self, request, context):
try:
[buildstream] 09/19: WIP: Temporarily disable source determinism
tests
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 110cc5d37cab61166021c6332876fd32b98e03bf
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Wed Nov 6 13:38:31 2019 +0000
WIP: Temporarily disable source determinism tests
These mess with umask and I don't want to think about them yet.
---
src/buildstream/testing/_sourcetests/source_determinism.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py
index fc0e461..724b24e 100644
--- a/src/buildstream/testing/_sourcetests/source_determinism.py
+++ b/src/buildstream/testing/_sourcetests/source_determinism.py
@@ -51,6 +51,7 @@ def create_test_directory(*path, mode=0o644):
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
@pytest.mark.skipif(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox, Must Fix')
+@pytest.mark.xfail
def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
project = str(datafiles)
element_name = 'list.bst'
[buildstream] 18/19: Make junction test subproject configuration
group-readable
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f5854eee0cf32462b25ea02b42c805453873f4a4
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Mon Nov 11 13:44:45 2019 +0000
Make junction test subproject configuration group-readable
---
src/buildstream/testing/_sourcetests/track_cross_junction.py | 3 +++
tests/frontend/track.py | 1 +
2 files changed, 4 insertions(+)
diff --git a/src/buildstream/testing/_sourcetests/track_cross_junction.py b/src/buildstream/testing/_sourcetests/track_cross_junction.py
index 550f57f..c9f82dd 100644
--- a/src/buildstream/testing/_sourcetests/track_cross_junction.py
+++ b/src/buildstream/testing/_sourcetests/track_cross_junction.py
@@ -22,6 +22,7 @@
import os
import pytest
+import stat
from buildstream import _yaml
from .._utils import generate_junction
@@ -79,6 +80,8 @@ def generate_project(tmpdir, name, kind, config=None):
}
project_conf.update(config)
_yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, 'project.conf'))
+ os.chmod(os.path.join(subproject_path, 'project.conf'),
+ stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP)
add_plugins_conf(subproject_path, kind)
return project_name, subproject_path
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index a628043..f30d45e 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -28,6 +28,7 @@ def generate_element(repo, element_path, dep_name=None):
element['depends'] = [dep_name]
_yaml.roundtrip_dump(element, element_path)
+ os.chmod(element_path, stat.S_IRUSR | stat.S_IRGRP)
@pytest.mark.datafiles(DATA_DIR)