You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:46:18 UTC
[buildstream] 06/19: Remove newly unused API surfaces in CASCache
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tlater/casd-socket-permissions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c424e99d4315c2d12d7bc803c9f894f8ddbb5cea
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Wed Oct 16 17:14:15 2019 +0100
Remove newly unused API surfaces in CASCache
This also involves a number of changes to tests and other parts of the
codebase since they were hacking about wit API that shouldn't have
existed.
---
src/buildstream/_artifactcache.py | 6 +-
src/buildstream/_basecache.py | 54 ++++++++++++-
src/buildstream/_cas/cascache.py | 165 +-------------------------------------
src/buildstream/_exceptions.py | 9 +++
tests/sourcecache/fetch.py | 9 +--
tests/sourcecache/push.py | 6 +-
tests/testutils/artifactshare.py | 14 +++-
7 files changed, 84 insertions(+), 179 deletions(-)
diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py
index d9112cd..91225ea 100644
--- a/src/buildstream/_artifactcache.py
+++ b/src/buildstream/_artifactcache.py
@@ -21,7 +21,7 @@ import os
import grpc
from ._basecache import BaseCache
-from ._exceptions import ArtifactError, CASError, CASCacheError, CASRemoteError
+from ._exceptions import ArtifactError, CASError, CacheError, CASRemoteError
from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
artifact_pb2, artifact_pb2_grpc
@@ -203,8 +203,8 @@ class ArtifactCache(BaseCache):
#
def remove(self, ref):
try:
- self.cas.remove(ref, basedir=self.artifactdir)
- except CASCacheError as e:
+ self._remove_ref(ref, self.artifactdir)
+ except CacheError as e:
raise ArtifactError("{}".format(e)) from e
# diff():
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py
index fc2e924..69da0f9 100644
--- a/src/buildstream/_basecache.py
+++ b/src/buildstream/_basecache.py
@@ -17,6 +17,7 @@
# Raoul Hidalgo Charman <ra...@codethink.co.uk>
#
import os
+import errno
from fnmatch import fnmatch
from itertools import chain
from typing import TYPE_CHECKING
@@ -25,7 +26,7 @@ from . import utils
from . import _yaml
from ._cas import CASRemote
from ._message import Message, MessageType
-from ._exceptions import LoadError, RemoteError
+from ._exceptions import LoadError, RemoteError, CacheError
from ._remote import RemoteSpec, RemoteType
@@ -425,3 +426,54 @@ class BaseCache():
if not glob_expr or fnmatch(relative_path, glob_expr):
# Obtain the mtime (the time a file was last modified)
yield (os.path.getmtime(ref_path), relative_path)
+
+ # _remove_ref()
+ #
+ # Removes a ref.
+ #
+ # This also takes care of pruning away directories which can
+ # be removed after having removed the given ref.
+ #
+ # Args:
+ # ref (str): The ref to remove
+ # basedir (str): Path of base directory the ref is in
+ #
+ # Raises:
+ # (CASCacheError): If the ref didnt exist, or a system error
+ # occurred while removing it
+ #
+ def _remove_ref(self, ref, basedir):
+
+ # Remove the ref itself
+ refpath = os.path.join(basedir, ref)
+
+ try:
+ os.unlink(refpath)
+ except FileNotFoundError as e:
+ raise CacheError("Could not find ref '{}'".format(ref)) from e
+
+ # Now remove any leading directories
+
+ components = list(os.path.split(ref))
+ while components:
+ components.pop()
+ refdir = os.path.join(basedir, *components)
+
+ # Break out once we reach the base
+ if refdir == basedir:
+ break
+
+ try:
+ os.rmdir(refdir)
+ except FileNotFoundError:
+ # The parent directory did not exist, but it's
+ # parent directory might still be ready to prune
+ pass
+ except OSError as e:
+ if e.errno == errno.ENOTEMPTY:
+ # The parent directory was not empty, so we
+ # cannot prune directories beyond this point
+ break
+
+ # Something went wrong here
+ raise CacheError("System error while removing ref '{}': {}".format(ref, e)) from e
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 0227304..11f15bd 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -21,7 +21,6 @@
import itertools
import os
import stat
-import errno
import contextlib
import ctypes
import multiprocessing
@@ -75,7 +74,6 @@ class CASCache():
):
self.casdir = os.path.join(path, 'cas')
self.tmpdir = os.path.join(path, 'tmp')
- os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
os.makedirs(self.tmpdir, exist_ok=True)
@@ -179,9 +177,7 @@ class CASCache():
# Preflight check.
#
def preflight(self):
- headdir = os.path.join(self.casdir, 'refs', 'heads')
- objdir = os.path.join(self.casdir, 'objects')
- if not (os.path.isdir(headdir) and os.path.isdir(objdir)):
+ if not os.path.join(self.casdir, 'objects'):
raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir))
# has_open_grpc_channels():
@@ -216,21 +212,6 @@ class CASCache():
self._terminate_casd_process(messenger)
shutil.rmtree(self._casd_socket_tempdir)
- # contains():
- #
- # Check whether the specified ref is already available in the local CAS cache.
- #
- # Args:
- # ref (str): The ref to check
- #
- # Returns: True if the ref is in the cache, False otherwise
- #
- def contains(self, ref):
- refpath = self._refpath(ref)
-
- # This assumes that the repository doesn't have any dangling pointers
- return os.path.exists(refpath)
-
# contains_file():
#
# Check whether a digest corresponds to a file which exists in CAS
@@ -309,28 +290,6 @@ class CASCache():
fullpath = os.path.join(dest, symlinknode.name)
os.symlink(symlinknode.target, fullpath)
- # diff():
- #
- # Return a list of files that have been added or modified between
- # the refs described by ref_a and ref_b.
- #
- # Args:
- # ref_a (str): The first ref
- # ref_b (str): The second ref
- # subdir (str): A subdirectory to limit the comparison to
- #
- def diff(self, ref_a, ref_b):
- tree_a = self.resolve_ref(ref_a)
- tree_b = self.resolve_ref(ref_b)
-
- added = []
- removed = []
- modified = []
-
- self.diff_trees(tree_a, tree_b, added=added, removed=removed, modified=modified)
-
- return modified, removed, added
-
# pull_tree():
#
# Pull a single Tree rather than a ref.
@@ -457,74 +416,6 @@ class CASCache():
return utils._message_digest(root_directory)
- # set_ref():
- #
- # Create or replace a ref.
- #
- # Args:
- # ref (str): The name of the ref
- #
- def set_ref(self, ref, tree):
- refpath = self._refpath(ref)
- os.makedirs(os.path.dirname(refpath), exist_ok=True)
- with utils.save_file_atomic(refpath, 'wb', tempdir=self.tmpdir) as f:
- f.write(tree.SerializeToString())
-
- # resolve_ref():
- #
- # Resolve a ref to a digest.
- #
- # Args:
- # ref (str): The name of the ref
- # update_mtime (bool): Whether to update the mtime of the ref
- #
- # Returns:
- # (Digest): The digest stored in the ref
- #
- def resolve_ref(self, ref, *, update_mtime=False):
- refpath = self._refpath(ref)
-
- try:
- with open(refpath, 'rb') as f:
- if update_mtime:
- os.utime(refpath)
-
- digest = remote_execution_pb2.Digest()
- digest.ParseFromString(f.read())
- return digest
-
- except FileNotFoundError as e:
- raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
-
- # update_mtime()
- #
- # Update the mtime of a ref.
- #
- # Args:
- # ref (str): The ref to update
- #
- def update_mtime(self, ref):
- try:
- os.utime(self._refpath(ref))
- except FileNotFoundError as e:
- raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
-
- # remove():
- #
- # Removes the given symbolic ref from the repo.
- #
- # Args:
- # ref (str): A symbolic ref
- # basedir (str): Path of base directory the ref is in, defaults to
- # CAS refs heads
- #
- def remove(self, ref, *, basedir=None):
-
- if basedir is None:
- basedir = os.path.join(self.casdir, 'refs', 'heads')
- # Remove cache ref
- self._remove_ref(ref, basedir)
-
def update_tree_mtime(self, tree):
reachable = set()
self._reachable_refs_dir(reachable, tree, update_mtime=True)
@@ -702,60 +593,6 @@ class CASCache():
return os.path.join(log_dir, str(self._casd_start_time) + ".log")
- def _refpath(self, ref):
- return os.path.join(self.casdir, 'refs', 'heads', ref)
-
- # _remove_ref()
- #
- # Removes a ref.
- #
- # This also takes care of pruning away directories which can
- # be removed after having removed the given ref.
- #
- # Args:
- # ref (str): The ref to remove
- # basedir (str): Path of base directory the ref is in
- #
- # Raises:
- # (CASCacheError): If the ref didnt exist, or a system error
- # occurred while removing it
- #
- def _remove_ref(self, ref, basedir):
-
- # Remove the ref itself
- refpath = os.path.join(basedir, ref)
-
- try:
- os.unlink(refpath)
- except FileNotFoundError as e:
- raise CASCacheError("Could not find ref '{}'".format(ref)) from e
-
- # Now remove any leading directories
-
- components = list(os.path.split(ref))
- while components:
- components.pop()
- refdir = os.path.join(basedir, *components)
-
- # Break out once we reach the base
- if refdir == basedir:
- break
-
- try:
- os.rmdir(refdir)
- except FileNotFoundError:
- # The parent directory did not exist, but it's
- # parent directory might still be ready to prune
- pass
- except OSError as e:
- if e.errno == errno.ENOTEMPTY:
- # The parent directory was not empty, so we
- # cannot prune directories beyond this point
- break
-
- # Something went wrong here
- raise CASCacheError("System error while removing ref '{}': {}".format(ref, e)) from e
-
def _get_subdir(self, tree, subdir):
head, name = os.path.split(subdir)
if head:
diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py
index 947b831..319518f 100644
--- a/src/buildstream/_exceptions.py
+++ b/src/buildstream/_exceptions.py
@@ -275,6 +275,15 @@ class SandboxError(BstError):
# SourceCacheError
#
+# Raised when errors are encountered in either type of cache
+#
+class CacheError(BstError):
+ def __init__(self, message, detail=None, reason=None):
+ super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason)
+
+
+# SourceCacheError
+#
# Raised when errors are encountered in the source caches
#
class SourceCacheError(BstError):
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index a5863b8..48683d6 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -92,8 +92,7 @@ def test_source_fetch(cli, tmpdir, datafiles):
assert not element._source_cached()
source = list(element.sources())[0]
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not share.get_source_proto(source._get_source_name())
# Just check that we sensibly fetch and build the element
res = cli.run(project=project_dir, args=['build', element_name])
@@ -139,8 +138,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
assert not element._source_cached()
source = list(element.sources())[0]
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not share.get_source_proto(source._get_source_name())
assert not os.path.exists(os.path.join(cache_dir, 'sources'))
# Now check if it falls back to the source fetch method.
@@ -198,8 +196,7 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
assert not element._source_cached()
source = list(element.sources())[0]
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not share.get_artifact_proto(source._get_source_name())
# Just check that we sensibly fetch and build the element
res = cli.run(project=project_dir, args=['build', element_name])
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 406aeba..8c0ac06 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -98,8 +98,7 @@ def test_source_push_split(cli, tmpdir, datafiles):
source = list(element.sources())[0]
# check we don't have it in the current cache
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not index.get_source_proto(source._get_source_name())
# build the element, this should fetch and then push the source to the
# remote
@@ -156,8 +155,7 @@ def test_source_push(cli, tmpdir, datafiles):
source = list(element.sources())[0]
# check we don't have it in the current cache
- cas = context.get_cascache()
- assert not cas.contains(source._get_source_name())
+ assert not share.get_source_proto(source._get_source_name())
# build the element, this should fetch and then push the source to the
# remote
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index d86cafa..ba02d39 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -11,7 +11,7 @@ from buildstream._cas import CASCache
from buildstream._cas.casserver import create_server
from buildstream._exceptions import CASError
from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
-from buildstream._protos.buildstream.v2 import artifact_pb2
+from buildstream._protos.buildstream.v2 import artifact_pb2, source_pb2
# ArtifactShare()
@@ -42,6 +42,7 @@ class ArtifactShare():
self.cas = CASCache(self.repodir, casd=casd)
self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs')
+ self.sourcedir = os.path.join(self.repodir, 'source_protos', 'refs')
self.quota = quota
self.index_only = index_only
@@ -127,6 +128,17 @@ class ArtifactShare():
return artifact_proto
+ def get_source_proto(self, source_name):
+ source_proto = source_pb2.Source()
+ source_path = os.path.join(self.sourcedir, source_name)
+
+ try:
+ with open(source_path, 'rb') as f:
+ source_proto.ParseFromString(f.read())
+ except FileNotFoundError:
+ return None
+ return source_proto
+
def get_cas_files(self, artifact_proto):
reachable = set()